public SandboxedProcessReports(
FileAccessManifest manifest,
PathTable pathTable,
long pipSemiStableHash,
string pipDescription,
LoggingContext loggingContext,
[CanBeNull] IDetoursEventListener detoursEventListener,
[CanBeNull] SidebandWriter sharedOpaqueOutputLogger)
{
Contract.Requires(manifest != null);
Contract.Requires(pathTable != null);
Contract.Requires(pipDescription != null);
PipSemiStableHash = pipSemiStableHash;
PipDescription = pipDescription;
m_pathTable = pathTable;
FileAccesses = manifest.ReportFileAccesses ? new HashSet <ReportedFileAccess>() : null;
FileUnexpectedAccesses = new HashSet <ReportedFileAccess>();
m_manifest = manifest;
m_detoursEventListener = detoursEventListener;
m_sharedOpaqueOutputLogger = sharedOpaqueOutputLogger;
// For tests we need the StaticContext
m_loggingContext = loggingContext ?? BuildXL.Utilities.Tracing.Events.StaticContext;
}
private void InvalidateSidebandFile(string sidebandFile, SidebandIntegrityCheckFailReason kind)
{
XAssert.FileExists(sidebandFile, "Sideband file for pipA not found.");
switch (kind)
{
case SidebandIntegrityCheckFailReason.FileNotFound:
AssertDeleteFile(sidebandFile, "Could not delete sideband file for pipA.");
break;
case SidebandIntegrityCheckFailReason.ChecksumMismatch:
File.WriteAllText(path: sidebandFile, contents: "bogus sideband file");
break;
case SidebandIntegrityCheckFailReason.MetadataMismatch:
SidebandMetadata alteredMetadata;
// read the header and the metadata from the original
using (var reader = new SidebandReader(sidebandFile))
{
XAssert.IsTrue(reader.ReadHeader(ignoreChecksum: false));
var originalMetadata = reader.ReadMetadata();
alteredMetadata = new SidebandMetadata(originalMetadata.PipSemiStableHash + 1, originalMetadata.StaticPipFingerprint);
}
// overwrite the original with a different metadata file where PiPSemiStableHash is different than in the original
using (var writer = new SidebandWriter(alteredMetadata, sidebandFile, null))
{
writer.EnsureHeaderWritten();
}
break;
default:
XAssert.Fail($"Unknown kind: {kind}");
break;
}
}
/// <summary>
/// Creates instance
/// </summary>
public SandboxedProcessInfo(
PathTable pathTable,
ISandboxedProcessFileStorage fileStorage,
string fileName,
FileAccessManifest fileAccessManifest,
bool disableConHostSharing,
ContainerConfiguration containerConfiguration,
bool testRetries = false,
LoggingContext loggingContext = null,
IDetoursEventListener detoursEventListener = null,
ISandboxConnection sandboxConnection = null,
SidebandWriter sidebandWriter = null)
{
Contract.Requires(pathTable != null);
Contract.Requires(fileStorage != null);
Contract.Requires(fileName != null);
PathTable = pathTable;
FileAccessManifest = fileAccessManifest;
FileStorage = fileStorage;
FileName = fileName;
DisableConHostSharing = disableConHostSharing;
// This should be set for testing purposes only.
TestRetries = testRetries;
NestedProcessTerminationTimeout = DefaultNestedProcessTerminationTimeout;
LoggingContext = loggingContext;
DetoursEventListener = detoursEventListener;
SandboxConnection = sandboxConnection;
ContainerConfiguration = containerConfiguration;
SidebandWriter = sidebandWriter;
}
public void DeserializeIsInverseOfSerialize(string logFile, string[] rootDirs)
{
using (var original = new SidebandWriter(DefaultSidebandMetadata, logFile, rootDirs))
using (var clone = CloneViaSerialization(original))
{
XAssert.AreEqual(original.SidebandLogFile, clone.SidebandLogFile);
XAssert.ArrayEqual(original.RootDirectories?.ToArray(), clone.RootDirectories?.ToArray());
}
}
public void ReadingFromCorruptedSidebandFiles(
int numValidRecordsToWriteFirst,
bool closeSidebandWriterCleanly,
bool appendBogusBytes)
{
var myDir = Path.Combine(TemporaryDirectory, nameof(ReadingFromCorruptedSidebandFiles));
Directory.CreateDirectory(myDir);
XAssert.IsTrue(Directory.Exists(myDir));
// write some valid records first
var validRecords = Enumerable
.Range(0, numValidRecordsToWriteFirst)
.Select(i => Path.Combine(TemporaryDirectory, $"path-{i}"))
.ToArray();
var sidebandFile = Path.Combine(myDir, $"bogus-log-close_{closeSidebandWriterCleanly}-append_{appendBogusBytes}");
var sidebandWriter = CreateWriter(sidebandFile);
sidebandWriter.EnsureHeaderWritten();
foreach (var path in validRecords)
{
XAssert.IsTrue(RecordFileWrite(sidebandWriter, path));
}
if (closeSidebandWriterCleanly)
{
sidebandWriter.Dispose();
}
else
{
sidebandWriter.CloseWriterWithoutFixingUpHeaderForTestingOnly();
}
if (appendBogusBytes)
{
// append some bogus stuff at the end
using (var s = new FileStream(sidebandFile, FileMode.OpenOrCreate))
using (var bw = new BinaryWriter(s))
{
bw.Seek(0, SeekOrigin.End);
bw.Write(1231);
bw.Flush();
}
}
// reading should produce valid records and just finish when it encounters the bogus stuff
var read = SidebandWriter.ReadRecordedPathsFromSidebandFile(sidebandFile).ToArray();
XAssert.ArrayEqual(validRecords, read);
}
private static SidebandWriter CloneViaSerialization(SidebandWriter original)
{
using (var stream = new MemoryStream())
{
using (var writer = new BuildXLWriter(debug: true, stream, leaveOpen: true, logStats: true))
{
original.Serialize(writer);
}
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new BuildXLReader(debug: true, stream, leaveOpen: true))
{
return(SidebandWriter.Deserialize(reader));
}
}
}
/// <summary>
/// Examines the state of the sideband files and returns all relevant information packaged up in a <see cref="Result"/> object.
/// </summary>
public Result Examine(bool computeExtraneousSidebandFiles)
{
try
{
if (!Configuration.Schedule.UnsafeLazySODeletion || !SidebandRootDir.IsValid)
{
return(Result.CreateForEagerDeletion());
}
// find relevant process pips (i.e., those with SOD outputs and not filtered out by RootFilter
var processesWithSharedOpaqueDirectoryOutputs = GetFilteredNodes()
.ToArray()
.AsParallel(Context)
.Select(nodeId => (Process)Scheduler.PipGraph.GetPipFromPipId(nodeId.ToPipId()))
.Where(process => process.HasSharedOpaqueDirectoryOutputs)
.ToArray();
// check validity of their sideband files
if (!processesWithSharedOpaqueDirectoryOutputs.All(process => ValidateSidebandFileForProcess(process)))
{
return(Result.CreateForEagerDeletion());
}
// find extraneous sideband files and return
string[] extraneousSidebandFiles = null;
if (computeExtraneousSidebandFiles)
{
var allSidebandFiles = SidebandWriter.FindAllProcessPipSidebandFiles(SidebandRootDir.ToString(Context.PathTable));
extraneousSidebandFiles = allSidebandFiles
.Except(
processesWithSharedOpaqueDirectoryOutputs.Select(GetSidebandFile),
StringComparer.OrdinalIgnoreCase)
.ToArray();
}
return(Result.CreateForLazyDeletion(extraneousSidebandFiles));
}
catch (IOException ex)
{
Logger.Log.SidebandFileIntegrityCheckThrewException(LoggingContext, ex.ToString());
return(Result.CreateForEagerDeletion());
}
}
/// <summary>
/// Creates instance
/// </summary>
public SandboxedProcessInfo(
PathTable pathTable,
[CanBeNull] ISandboxedProcessFileStorage fileStorage,
string fileName,
FileAccessManifest fileAccessManifest,
bool disableConHostSharing,
ContainerConfiguration containerConfiguration,
LoggingContext loggingContext,
bool testRetries = false,
IDetoursEventListener detoursEventListener = null,
ISandboxConnection sandboxConnection = null,
SidebandWriter sidebandWriter = null,
bool createJobObjectForCurrentProcess = true,
ISandboxFileSystemView fileSystemView = null,
SandboxedProcessResourceMonitoringConfig monitoringConfig = null)
{
Contract.RequiresNotNull(pathTable);
Contract.RequiresNotNull(fileName);
PathTable = pathTable;
FileAccessManifest = fileAccessManifest;
FileStorage = fileStorage;
FileName = fileName;
DisableConHostSharing = disableConHostSharing;
// This should be set for testing purposes only.
TestRetries = testRetries;
NestedProcessTerminationTimeout = DefaultNestedProcessTerminationTimeout;
LoggingContext = loggingContext;
DetoursEventListener = detoursEventListener;
SandboxConnection = sandboxConnection;
ContainerConfiguration = containerConfiguration;
SidebandWriter = sidebandWriter;
CreateJobObjectForCurrentProcess = createJobObjectForCurrentProcess;
FileSystemView = fileSystemView;
MonitoringConfig = monitoringConfig;
}
public SandboxedProcessReports(
FileAccessManifest manifest,
PathTable pathTable,
long pipSemiStableHash,
string pipDescription,
LoggingContext loggingContext,
[CanBeNull] IDetoursEventListener detoursEventListener,
[CanBeNull] SidebandWriter sharedOpaqueOutputLogger)
{
Contract.RequiresNotNull(manifest);
Contract.RequiresNotNull(pathTable);
Contract.RequiresNotNull(pipDescription);
Contract.RequiresNotNull(loggingContext);
PipSemiStableHash = pipSemiStableHash;
PipDescription = pipDescription;
m_pathTable = pathTable;
FileAccesses = manifest.ReportFileAccesses ? new HashSet <ReportedFileAccess>() : null;
FileUnexpectedAccesses = new HashSet <ReportedFileAccess>();
m_manifest = manifest;
m_detoursEventListener = detoursEventListener;
m_sharedOpaqueOutputLogger = sharedOpaqueOutputLogger;
m_loggingContext = loggingContext;
}
private bool RecordFileWrite(SidebandWriter writer, string path) => writer.RecordFileWrite(PathTable, path, flushImmediately: false);
private string GetSidebandFile(Process process) => SidebandWriter.GetSidebandFileForProcess(Context.PathTable, SidebandRootDir, process);
public void SerializeSandboxedProcessInfo(bool useNullFileStorage)
{
var pt = new PathTable();
var fam =
new FileAccessManifest(pt, CreateDirectoryTranslator())
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false
};
var vac = new ValidationDataCreator(fam, pt);
vac.AddScope(A("C", "Users", "AppData"), FileAccessPolicy.AllowAll);
vac.AddPath(A("C", "Source", "source.txt"), FileAccessPolicy.AllowReadAlways);
vac.AddPath(A("C", "Out", "out.txt"), FileAccessPolicy.AllowAll);
SandboxedProcessStandardFiles standardFiles = null;
ISandboxedProcessFileStorage fileStorage;
if (useNullFileStorage)
{
fileStorage = null;
}
else
{
standardFiles = new SandboxedProcessStandardFiles(A("C", "pip", "pip.out"), A("C", "pip", "pip.err"));
fileStorage = new StandardFileStorage(standardFiles);
}
var envVars = new Dictionary <string, string>()
{
["Var1"] = "Val1",
["Var2"] = "Val2",
};
IBuildParameters buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars);
var sidebandLogFile = A("C", "engine-cache", "sideband-logs", "log-1");
var loggerRootDirs = new[] { A("C", "out", "dir1"), A("C", "out", "dir2") };
var sharedOpaqueOutputLogger = new SidebandWriter(DefaultSidebandMetadata, sidebandLogFile, loggerRootDirs);
SandboxedProcessInfo info = new SandboxedProcessInfo(
pt,
fileStorage,
A("C", "tool", "tool.exe"),
fam,
true,
null,
sidebandWriter: sharedOpaqueOutputLogger)
{
Arguments = @"/arg1:val1 /arg2:val2",
WorkingDirectory = A("C", "Source"),
EnvironmentVariables = buildParameters,
Timeout = TimeSpan.FromMinutes(15),
PipSemiStableHash = 0x12345678,
PipDescription = nameof(SerializeSandboxedProcessInfo),
TimeoutDumpDirectory = A("C", "Timeout"),
SandboxKind = global::BuildXL.Utilities.Configuration.SandboxKind.Default,
AllowedSurvivingChildProcessNames = new[] { "conhost.exe", "mspdbsrv.exe" },
NestedProcessTerminationTimeout = SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
StandardInputSourceInfo = StandardInputInfo.CreateForData("Data"),
StandardObserverDescriptor = new SandboxObserverDescriptor()
{
WarningRegex = new ExpandedRegexDescriptor("*warn", System.Text.RegularExpressions.RegexOptions.Compiled)
},
};
// Serialize and deserialize.
SandboxedProcessInfo readInfo = null;
using (var stream = new MemoryStream())
{
info.Serialize(stream);
stream.Position = 0;
readInfo = SandboxedProcessInfo.Deserialize(
stream,
new global::BuildXL.Utilities.Instrumentation.Common.LoggingContext("Test"),
null);
}
using (readInfo.SidebandWriter)
{
// Verify.
XAssert.AreEqual(info.FileName, readInfo.FileName);
XAssert.AreEqual(info.Arguments, readInfo.Arguments);
XAssert.AreEqual(info.WorkingDirectory, readInfo.WorkingDirectory);
var readEnvVars = readInfo.EnvironmentVariables.ToDictionary();
XAssert.AreEqual(envVars.Count, readEnvVars.Count);
foreach (var kvp in envVars)
{
XAssert.AreEqual(kvp.Value, readEnvVars[kvp.Key]);
}
XAssert.AreEqual(info.Timeout, readInfo.Timeout);
XAssert.AreEqual(info.PipSemiStableHash, readInfo.PipSemiStableHash);
XAssert.AreEqual(info.PipDescription, readInfo.PipDescription);
XAssert.AreEqual(info.TimeoutDumpDirectory, readInfo.TimeoutDumpDirectory);
XAssert.AreEqual(info.SandboxKind, readInfo.SandboxKind);
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames.Length, readInfo.AllowedSurvivingChildProcessNames.Length);
for (int i = 0; i < info.AllowedSurvivingChildProcessNames.Length; ++i)
{
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames[i], readInfo.AllowedSurvivingChildProcessNames[i]);
}
XAssert.AreEqual(info.NestedProcessTerminationTimeout, readInfo.NestedProcessTerminationTimeout);
XAssert.AreEqual(info.StandardInputSourceInfo, readInfo.StandardInputSourceInfo);
if (useNullFileStorage)
{
XAssert.IsNull(readInfo.SandboxedProcessStandardFiles);
XAssert.IsNull(readInfo.FileStorage);
}
else
{
XAssert.IsNotNull(readInfo.SandboxedProcessStandardFiles);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.SandboxedProcessStandardFiles.StandardOutput);
XAssert.AreEqual(standardFiles.StandardError, readInfo.SandboxedProcessStandardFiles.StandardError);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
XAssert.AreEqual(standardFiles.StandardError, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardError));
}
XAssert.IsFalse(readInfo.ContainerConfiguration.IsIsolationEnabled);
XAssert.AreEqual(sidebandLogFile, readInfo.SidebandWriter.SidebandLogFile);
XAssert.ArrayEqual(loggerRootDirs, readInfo.SidebandWriter.RootDirectories.ToArray());
if (!OperatingSystemHelper.IsUnixOS)
{
// this validator examines serialized FAM bytes using the same Windows-only native parser used by Detours
ValidationDataCreator.TestManifestRetrieval(vac.DataItems, readInfo.FileAccessManifest, false);
}
}
}
