public static ReportedFileAccess Create(
ReportedFileOperation operation,
ReportedProcess process,
RequestedAccess requestedAccess,
FileAccessStatus status,
bool explicitlyReported,
uint error,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
FlagsAndAttributes openedFileOrDirectoryAttribute,
PathTable pathTable,
string path,
string enumeratePattern = null)
{
Contract.Requires(process != null);
Contract.Requires(pathTable != null);
Contract.Requires(path != null);
AbsolutePath absolutePath;
if (AbsolutePath.TryGet(pathTable, (StringSegment)path, out absolutePath))
{
return(new ReportedFileAccess(
operation,
process,
requestedAccess,
status,
explicitlyReported,
error,
usn,
desiredAccess,
shareMode,
creationDisposition,
flagsAndAttributes,
openedFileOrDirectoryAttribute,
absolutePath,
null,
enumeratePattern));
}
return(new ReportedFileAccess(
operation,
process,
requestedAccess,
status,
explicitlyReported,
error,
usn,
desiredAccess,
shareMode,
creationDisposition,
flagsAndAttributes,
openedFileOrDirectoryAttribute,
AbsolutePath.Invalid,
path,
enumeratePattern));
}
public Entry(Usn usn, ContentHash hash, long length, byte timeToLive)
{
Hash = hash;
TimeToLive = timeToLive;
Usn = usn;
Length = length;
}
public async Task WriteAllBytesCallback()
{
const string Target = "target";
Usn closeUsn = default(Usn);
await FileUtilities.WriteAllBytesAsync(
GetFullPath(Target),
Encoding.UTF8.GetBytes("Target"),
onCompletion : stream =>
{
Usn?maybeCloseUsn = FileUtilities.TryWriteUsnCloseRecordByHandle(stream);
XAssert.IsNotNull(maybeCloseUsn);
closeUsn = maybeCloseUsn.Value;
});
XAssert.IsTrue(File.Exists(GetFullPath(Target)));
XAssert.AreEqual("Target", File.ReadAllText(GetFullPath(Target)));
XAssert.AreNotEqual(0, closeUsn, "Completion callback skipped");
using (FileStream dest = File.OpenRead(GetFullPath(Target)))
{
Usn usn = FileUtilities.ReadFileUsnByHandle(dest.SafeFileHandle).Value.Usn;
XAssert.AreEqual(closeUsn, usn, "CLOSE usn should have been written during the callback.");
}
}
public ReportedFileAccess(
ReportedFileOperation operation,
ReportedProcess process,
RequestedAccess requestedAccess,
FileAccessStatus status,
bool explicitlyReported,
uint error,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
AbsolutePath manifestPath,
string path,
string enumeratePatttern)
{
Contract.Requires(process != null);
Operation = operation;
Process = process;
RequestedAccess = requestedAccess;
Status = status;
ExplicitlyReported = explicitlyReported;
Error = error;
Usn = usn;
DesiredAccess = desiredAccess;
ShareMode = shareMode;
CreationDisposition = creationDisposition;
FlagsAndAttributes = flagsAndAttributes;
ManifestPath = manifestPath;
Path = path;
EnumeratePattern = enumeratePatttern;
}
public static ReportedFileAccess Create(
ReportedFileOperation operation,
ReportedProcess process,
RequestedAccess requestedAccess,
FileAccessStatus status,
bool explicitlyReported,
uint error,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
AbsolutePath path,
string enumeratePattern = null)
{
Contract.Requires(process != null);
return(new ReportedFileAccess(
operation,
process,
requestedAccess,
status,
explicitlyReported,
error,
usn,
desiredAccess,
shareMode,
creationDisposition,
flagsAndAttributes,
path,
null,
enumeratePattern));
}
/// <inheritdoc />
public unsafe ReadUsnJournalResult TryReadUsnJournal(
SafeFileHandle volumeHandle,
byte[] buffer,
ulong journalId,
Usn startUsn = default(Usn),
bool forceJournalVersion2 = false,
bool isJournalUnprivileged = false) => throw new NotImplementedException();
/// <nodoc />
public ReadJournalResponse(
Usn nextUsn,
ReadUsnJournalStatus status,
bool timeout = false)
{
NextUsn = nextUsn;
Status = status;
Timeout = timeout;
}
/// <summary>
/// Constructor for USN-based identity.
/// </summary>
public VersionedFileIdentity(ulong volumeSerialNumber, FileId fileId, Usn usn, IdentityKind kind)
{
Contract.Requires(kind.IsWeakOrStrong());
m_fileId = fileId;
m_volumeSerialNumber = volumeSerialNumber;
m_usn = usn;
Kind = kind;
}
private void ExpectChangesSinceUsn(
UsnChangeReasons expectedChangeReasons,
SafeFileHandle volumeHandle,
Usn startUsn,
FileId fileId,
out Usn nextUsn,
TimeSpan?timeLimit = default(TimeSpan?))
{
const int DefaultTimeLimitForScanningInSecond = 30; // 30 sec for scanning.
QueryUsnJournalData journalState = QueryJournal(volumeHandle);
byte[] buffer = new byte[64 * 1024]; // 655 records per read.
timeLimit = timeLimit.HasValue ? timeLimit : TimeSpan.FromSeconds(DefaultTimeLimitForScanningInSecond);
var stopWatch = System.Diagnostics.Stopwatch.StartNew();
UsnChangeReasons foundChangeReasons = 0;
nextUsn = startUsn;
while (true)
{
if (stopWatch.ElapsedTicks > timeLimit.Value.Ticks)
{
break;
}
ReadUsnJournalResult result = FileUtilities.TryReadUsnJournal(
volumeHandle,
buffer,
journalState.UsnJournalId,
startUsn);
nextUsn = result.NextUsn;
if (!result.Succeeded)
{
break;
}
if (result.Records.Count == 0)
{
break;
}
foundChangeReasons |= UsnJournalUtilities.GetAggregateChangeReasons(fileId, result.Records);
if (expectedChangeReasons == (foundChangeReasons & expectedChangeReasons))
{
// Found all expected change reasons.
return;
}
startUsn = result.NextUsn;
}
XAssert.AreEqual(expectedChangeReasons, foundChangeReasons & expectedChangeReasons);
}
private void DoReportAccess(
ReportedFileOperation reportedFileOperation,
RequestedAccess requestedAccess,
FileAccessStatus fileAccessStatus,
uint errorCode,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
FlagsAndAttributes openedFileOrDirectoryAttributes,
string path,
string enumeratePattern,
string processArgs)
{
var process = Process.GetCurrentProcess();
Contract.Requires(!string.IsNullOrEmpty(path));
Contract.Requires(Path.IsPathRooted(path), $"Provided path is expected to be rooted, but found '{path}'.");
// The given path may not be canonicalized (e.g. it may contain '..')
string fullPath;
try
{
fullPath = FileUtilities.GetFullPath(path);
}
catch (BuildXLException)
{
// If getting the full path fails, we follow a behavior similar to what detours does, where the access is ignored
return;
}
string access = FileAccessReportLine.GetReportLineForAugmentedFileAccess(
reportedFileOperation,
(uint)process.Id,
requestedAccess,
fileAccessStatus,
errorCode,
usn,
desiredAccess,
shareMode,
creationDisposition,
flagsAndAttributes,
openedFileOrDirectoryAttributes,
fullPath,
enumeratePattern,
processArgs);
if (!FileUtilities.TryWriteFileSync(m_detoursReportHandle, m_encoding.GetBytes(access), out int nativeErrorCode))
{
// Something didn't go as expected. We cannot let the process continue if we failed at reporting an access
throw new NativeWin32Exception(nativeErrorCode, $"Writing augmented file access report failed. Line: {access}");
}
}
private bool ReportProvider(
ref AccessReport report, out uint processId, out uint id, out uint correlationId, out ReportedFileOperation operation, out RequestedAccess requestedAccess, out FileAccessStatus status,
out bool explicitlyReported, out uint error, out Usn usn, out DesiredAccess desiredAccess, out ShareMode shareMode, out CreationDisposition creationDisposition,
out FlagsAndAttributes flagsAndAttributes, out FlagsAndAttributes openedFileOrDirectoryAttributes, out AbsolutePath manifestPath, out string path, out string enumeratePattern, out string processArgs, out string errorMessage)
{
var errorMessages = new List <string>();
checked
{
processId = (uint)report.Pid;
id = SandboxedProcessReports.FileAccessNoId;
correlationId = SandboxedProcessReports.FileAccessNoId;
if (!FileAccessReportLine.Operations.TryGetValue(report.DecodeOperation(), out operation))
{
errorMessages.Add($"Unknown operation '{report.DecodeOperation()}'");
}
requestedAccess = (RequestedAccess)report.RequestedAccess;
if (report.RequestedAccess > s_maxRequestedAccess)
{
errorMessages.Add($"Illegal value for 'RequestedAccess': {requestedAccess}; maximum allowed: {(int)RequestedAccess.All}");
}
status = (FileAccessStatus)report.Status;
if (report.Status > s_maxFileAccessStatus)
{
errorMessages.Add($"Illegal value for 'Status': {status}");
}
bool isWrite = (report.RequestedAccess & (byte)RequestedAccess.Write) != 0;
explicitlyReported = report.ExplicitLogging > 0;
error = report.Error;
usn = ReportedFileAccess.NoUsn;
desiredAccess = isWrite ? DesiredAccess.GENERIC_WRITE : DesiredAccess.GENERIC_READ;
shareMode = ShareMode.FILE_SHARE_READ;
creationDisposition = CreationDisposition.OPEN_ALWAYS;
flagsAndAttributes = 0;
openedFileOrDirectoryAttributes = 0;
path = report.DecodePath();
enumeratePattern = string.Empty;
processArgs = string.Empty;
AbsolutePath.TryCreate(PathTable, path, out manifestPath);
errorMessage = errorMessages.Any()
? $"Illegal access report: '{AccessReportToString(report)}' :: {string.Join(";", errorMessages)}"
: string.Empty;
return(errorMessage == string.Empty);
}
}
/// <summary>
/// Returns a detours report line representing an augmented file access.
/// </summary>
/// <remarks>
/// The line format is compatible with a regular file access report line. This is not a hard requirement (since <see cref="ReportType.AugmentedFileAccess"/>
/// is used to distinguish this line from regular lines), but is convenient to use the same report line parser for all cases. So future changes
/// can happen here as long as they are kept in sync with <see cref="SandboxedProcessReports.TryParseAugmentedFileAccess"/>
/// </remarks>
internal static string GetReportLineForAugmentedFileAccess(
ReportedFileOperation reportedFileOperation,
uint processId,
RequestedAccess requestedAccess,
FileAccessStatus fileAccessStatus,
uint errorCode,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
FlagsAndAttributes openedFileOrDirectoryAttributes,
string absolutePath,
[CanBeNull] string enumeratePattern,
[CanBeNull] string processArgs)
{
var result = new System.Text.StringBuilder();
result.Append($"{(int)ReportType.AugmentedFileAccess},{reportedFileOperation}:");
result.Append($"{processId:x}|");
result.Append($"{SandboxedProcessReports.FileAccessNoId:x}|"); // no id.
result.Append($"{SandboxedProcessReports.FileAccessNoId:x}|"); // no correlation id.
result.Append($"{(byte)requestedAccess:x}|");
result.Append($"{(byte)fileAccessStatus:x}|");
// '1' makes the access look as explicitly reported, but this actually doesn't matter since it will get
// set based on the manifest policy upon reception
result.Append("1|");
result.Append($"{errorCode:x}|");
result.Append($"{usn.Value:x}|");
result.Append($"{(uint)desiredAccess:x}|");
result.Append($"{(uint)shareMode:x}|");
result.Append($"{(uint)creationDisposition:x}|");
result.Append($"{(uint)flagsAndAttributes:x}|");
result.Append($"{(uint)openedFileOrDirectoryAttributes:x}|");
// The manifest path is always written as invalid
result.Append($"{AbsolutePath.Invalid.Value.Value:x}|");
result.Append(absolutePath);
if (!string.IsNullOrEmpty(enumeratePattern))
{
Contract.Assert(requestedAccess == RequestedAccess.Enumerate);
result.Append($"|{enumeratePattern}");
}
if (!string.IsNullOrEmpty(processArgs))
{
Contract.Assert(reportedFileOperation == ReportedFileOperation.Process);
result.Append($"|{processArgs}");
}
return(result.Append("\r\n").ToString());
}
public void QueryJournal()
{
WithVolumeHandle(
volumeHandle =>
{
using (FileStream file = File.Create(GetFullPath("File")))
{
QueryUsnJournalData journalState = QueryJournal(volumeHandle);
Usn usn = FileUtilities.ReadFileUsnByHandle(file.SafeFileHandle).Value.Usn;
XAssert.IsTrue(journalState.LowestValidUsn <= usn);
XAssert.IsTrue(journalState.NextUsn > usn);
}
});
}
/// <nodoc />
public IncorrectFileContentEntry(
string path,
FileIdAndVolumeId fileIdAndVolumeId,
Usn usn,
ContentHash expectedHash,
ContentHash actualHash)
{
Contract.Requires(expectedHash != actualHash);
Contract.Requires(!string.IsNullOrEmpty(path));
Path = path;
FileIdAndVolumeId = fileIdAndVolumeId;
Usn = usn;
ExpectedHash = expectedHash;
ActualHash = actualHash;
}
/// <summary>
/// Returns a detours report line representing an augmented file access.
/// </summary>
/// <remarks>
/// The line format is compatible with a regular file access report line. This is not a hard requirement (since <see cref="ReportType.AugmentedFileAccess"/>
/// is used to distinguish this line from regular lines), but is convenient to use the same report line parser for all cases. So future changes
/// can happen here as long as they are kept in sync with <see cref="SandboxedProcessReports.TryParseAugmentedFileAccess"/>
/// </remarks>
internal static string GetReportLineForAugmentedFileAccess(
ReportedFileOperation reportedFileOperation,
uint processId,
RequestedAccess requestedAccess,
FileAccessStatus fileAccessStatus,
uint errorCode,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
string absolutePath,
[CanBeNull] string enumeratePattern,
[CanBeNull] string processArgs)
{
var result = new System.Text.StringBuilder();
result.Append($"{(int)ReportType.AugmentedFileAccess},{reportedFileOperation}:");
result.Append($"{processId.ToString("x")}|");
result.Append($"{((byte)requestedAccess).ToString("x")}|");
result.Append($"{((byte)fileAccessStatus).ToString("x")}|");
// '1' makes the access look as explicitly reported
result.Append("1|");
result.Append($"{errorCode.ToString("x")}|");
result.Append($"{usn.Value.ToString("x")}|");
result.Append($"{((uint)desiredAccess).ToString("x")}|");
result.Append($"{((uint)shareMode).ToString("x")}|");
result.Append($"{((uint)creationDisposition).ToString("x")}|");
result.Append($"{((uint)flagsAndAttributes).ToString("x")}|");
// The manifest path is always written as invalid
result.Append($"{AbsolutePath.Invalid.Value.Value.ToString("x")}|");
result.Append(absolutePath);
if (!string.IsNullOrEmpty(enumeratePattern))
{
Contract.Assert(requestedAccess == RequestedAccess.Enumerate);
result.Append($"|{enumeratePattern}");
}
if (!string.IsNullOrEmpty(processArgs))
{
Contract.Assert(reportedFileOperation == ReportedFileOperation.Process);
result.Append($"|{processArgs}");
}
return(result.Append("\r\n").ToString());
}
/// <nodoc />
public ReadJournalRequest(
VolumeGuidPath volumeGuidPath,
ulong journalId,
Usn startUsn,
Usn?endUsn = default(Usn?),
int?extraReadCount = default(int?),
TimeSpan?timeLimit = default(TimeSpan?))
{
Contract.Requires(volumeGuidPath.IsValid);
Contract.Requires(!extraReadCount.HasValue || extraReadCount >= 0);
m_volumeGuidPath = volumeGuidPath;
JournalId = journalId;
StartUsn = startUsn;
EndUsn = endUsn;
ExtraReadCount = extraReadCount;
TimeLimit = timeLimit;
}
public void QueryJournal()
{
WithVolumeHandle(
volumeHandle =>
{
using (FileStream file = File.Create(GetFullPath("File")))
{
QueryUsnJournalResult queryResult = FileUtilities.TryQueryUsnJournal(volumeHandle);
XAssert.AreEqual(
QueryUsnJournalStatus.Success,
queryResult.Status,
"Failed to query the volume's change journal. Is it enabled on the test output directory's volume?");
XAssert.IsTrue(queryResult.Succeeded);
Usn usn = FileUtilities.ReadFileUsnByHandle(file.SafeFileHandle).Value.Usn;
XAssert.IsTrue(queryResult.Data.LowestValidUsn <= usn);
XAssert.IsTrue(queryResult.Data.NextUsn > usn);
}
});
}
public async Task CopyCompletionCallback()
{
const string Src = "src";
const string Target = "target";
File.WriteAllText(GetFullPath(Src), "Source");
Usn closeUsn = default(Usn);
var completionHandlerCalled = false;
await FileUtilities.CopyFileAsync(
GetFullPath(Src),
GetFullPath(Target),
onCompletion : (source, dest) =>
{
completionHandlerCalled = true;
if (!OperatingSystemHelper.IsUnixOS)
{
Usn?maybeCloseUsn = FileUtilities.TryWriteUsnCloseRecordByHandle(dest);
XAssert.IsNotNull(maybeCloseUsn);
closeUsn = maybeCloseUsn.Value;
}
});
XAssert.IsTrue(completionHandlerCalled);
XAssert.IsTrue(File.Exists(GetFullPath(Target)));
XAssert.AreEqual("Source", File.ReadAllText(GetFullPath(Target)));
if (!OperatingSystemHelper.IsUnixOS)
{
XAssert.AreNotEqual(0, closeUsn, "Completion callback skipped");
using (FileStream dest = File.OpenRead(GetFullPath(Target)))
{
Usn usn = FileUtilities.ReadFileUsnByHandle(dest.SafeFileHandle).Value.Usn;
XAssert.AreEqual(closeUsn, usn, "CLOSE usn should have been written during the callback.");
}
}
}
/// <summary>
/// Attempts to report a file access to BuildXL without actually performing any IO
/// </summary>
/// <remarks>
/// Failure means this method was invoked from a process that was not configured to breakaway from the sandbox.
/// The provided path is required to be non-null and rooted
/// </remarks>
public bool TryReportAugmentedFileAccess(
ReportedFileOperation reportedFileOperation,
RequestedAccess requestedAccess,
FileAccessStatus fileAccessStatus,
uint errorCode,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
FlagsAndAttributes openedFileOrDirectoryAttributes,
string path,
[CanBeNull] string enumeratePattern = null,
[CanBeNull] string processArgs = null)
{
Contract.Requires(!string.IsNullOrEmpty(path));
if (m_detoursReportHandle == null)
{
return(false);
}
DoReportAccess(
reportedFileOperation,
requestedAccess,
fileAccessStatus,
errorCode,
usn,
desiredAccess,
shareMode,
creationDisposition,
flagsAndAttributes,
openedFileOrDirectoryAttributes,
path,
enumeratePattern,
processArgs);
return(true);
}
public void ReadChangesToFile()
{
string path = GetFullPath("File");
WithVolumeHandle(
volumeHandle =>
{
Usn initialUsn = QueryJournal(volumeHandle).NextUsn;
FileId fileId;
using (FileStream file = File.Create(path))
{
fileId = FileUtilities.ReadFileUsnByHandle(file.SafeFileHandle).Value.FileId;
file.WriteByte(1);
}
const UsnChangeReasons ExpectedCreationChangeReasons =
UsnChangeReasons.FileCreate |
UsnChangeReasons.DataExtend |
UsnChangeReasons.Close;
Usn nextUsn;
ExpectChangesSinceUsn(ExpectedCreationChangeReasons, volumeHandle, initialUsn, fileId, out nextUsn);
using (FileStream file = File.OpenWrite(path))
{
file.WriteByte(1);
}
File.SetAttributes(path, FileAttributes.Normal | FileAttributes.ReadOnly);
const UsnChangeReasons ExpectedModificationChangeReasons =
UsnChangeReasons.DataOverwrite |
UsnChangeReasons.BasicInfoChange |
UsnChangeReasons.Close;
ExpectChangesSinceUsn(ExpectedModificationChangeReasons, volumeHandle, nextUsn, fileId, out nextUsn);
});
}
private void DoReportAccess(
ReportedFileOperation reportedFileOperation,
RequestedAccess requestedAccess,
FileAccessStatus fileAccessStatus,
uint errorCode,
Usn usn,
DesiredAccess desiredAccess,
ShareMode shareMode,
CreationDisposition creationDisposition,
FlagsAndAttributes flagsAndAttributes,
string path,
string enumeratePattern,
string processArgs)
{
var process = Process.GetCurrentProcess();
string access = FileAccessReportLine.GetReportLineForAugmentedFileAccess(
reportedFileOperation,
(uint)process.Id,
requestedAccess,
fileAccessStatus,
errorCode,
usn,
desiredAccess,
shareMode,
creationDisposition,
flagsAndAttributes,
path,
enumeratePattern,
processArgs);
if (!FileUtilities.TryWriteFileSync(m_detoursReportHandle, m_encoding.GetBytes(access), out int nativeErrorCode))
{
// Something didn't go as expected. We cannot let the process continue if we failed at reporting an access
throw new NativeWin32Exception(nativeErrorCode, $"Writing augmented file access report failed. Line: {access}");
}
}
private static LoadResult TryLoadInternal(string fileContentTablePath, byte entryTimeToLive)
{
Contract.Requires(!string.IsNullOrWhiteSpace(fileContentTablePath));
Contract.Requires(entryTimeToLive > 0);
if (!FileUtilities.FileExistsNoFollow(fileContentTablePath))
{
return(LoadResult.FileNotFound(fileContentTablePath));
}
Stopwatch sw = Stopwatch.StartNew();
try
{
using (FileStream stream = FileUtilities.CreateFileStream(
fileContentTablePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete,
// Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point.
FileOptions.SequentialScan))
{
try
{
Analysis.IgnoreResult(s_fileEnvelope.ReadHeader(stream));
}
catch (BuildXLException ex)
{
return(LoadResult.InvalidFormat(fileContentTablePath, ex.LogEventMessage, sw.ElapsedMilliseconds));
}
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
var loadedTable = new FileContentTable();
uint numberOfEntries = reader.ReadUInt32();
int hashLength = ContentHashingUtilities.HashInfo.ByteLength;
var hashBuffer = new byte[hashLength];
for (uint i = 0; i < numberOfEntries; i++)
{
// Key: Volume and file ID
var fileIdAndVolumeId = FileIdAndVolumeId.Deserialize(reader);
// Entry: USN, hash, length, time to live.
Usn usn = new Usn(reader.ReadUInt64());
int hashBytesRead = 0;
while (hashBytesRead != hashLength)
{
int thisRead = reader.Read(hashBuffer, hashBytesRead, hashLength - hashBytesRead);
if (thisRead == 0)
{
return(LoadResult.InvalidFormat(fileContentTablePath, "Unexpected end of stream", sw.ElapsedMilliseconds));
}
hashBytesRead += thisRead;
Contract.Assert(hashBytesRead <= hashLength);
}
long length = reader.ReadInt64();
byte thisEntryTimeToLive = reader.ReadByte();
if (thisEntryTimeToLive == 0)
{
return(LoadResult.InvalidFormat(fileContentTablePath, "TTL value must be positive", sw.ElapsedMilliseconds));
}
thisEntryTimeToLive = Math.Min(thisEntryTimeToLive, entryTimeToLive);
Contract.Assert(thisEntryTimeToLive > 0);
// We've loaded this entry just now and so clearly haven't used it yet. Tentatively decrement the TTL
// for the in-memory table; if the table is saved again without using this entry, the TTL will stay at this
// lower value.
thisEntryTimeToLive--;
var observedVersionAndHash = new Entry(usn, ContentHashingUtilities.CreateFrom(hashBuffer), length, thisEntryTimeToLive);
bool added = loadedTable.m_entries.TryAdd(fileIdAndVolumeId, observedVersionAndHash);
Contract.Assume(added);
}
loadedTable.Counters.AddToCounter(FileContentTableCounters.NumEntries, loadedTable.Count);
return(LoadResult.Success(fileContentTablePath, loadedTable, sw.ElapsedMilliseconds));
}
}
}
catch (Exception ex)
{
return(LoadResult.Exception(fileContentTablePath, ex, sw.ElapsedMilliseconds));
}
}
/// <inheritdoc />
public MaybeResponse <ReadJournalResponse> ReadJournal(ReadJournalRequest request, Action <UsnRecord> onUsnRecordReceived)
{
Contract.Assert(request.VolumeGuidPath.IsValid);
SafeFileHandle volumeHandle;
OpenFileResult volumeOpenResult = OpenVolumeHandle(request.VolumeGuidPath, out volumeHandle);
using (volumeHandle)
{
if (!volumeOpenResult.Succeeded)
{
string message = I($"Failed to open a volume handle for the volume '{request.VolumeGuidPath.Path}'");
return(new MaybeResponse <ReadJournalResponse>(
new ErrorResponse(ErrorStatus.FailedToOpenVolumeHandle, message)));
}
Usn startUsn = request.StartUsn;
Usn endUsn = request.EndUsn ?? Usn.Zero;
int extraReadCount = request.ExtraReadCount ?? -1;
long timeLimitInTicks = request.TimeLimit?.Ticks ?? -1;
var sw = new StopwatchVar();
using (var swRun = sw.Start())
{
byte[] buffer = new byte[JournalReadBufferSize];
while (true)
{
if (timeLimitInTicks >= 0 && timeLimitInTicks < swRun.ElapsedTicks)
{
return(new MaybeResponse <ReadJournalResponse>(
new ReadJournalResponse(status: ReadUsnJournalStatus.Success, nextUsn: startUsn, timeout: true)));
}
ReadUsnJournalResult result = FileUtilities.TryReadUsnJournal(
volumeHandle,
buffer,
request.JournalId,
startUsn,
isJournalUnprivileged: IsJournalUnprivileged);
if (!result.Succeeded)
{
// Bug #1164760 shows that the next USN can be non-zero.
return(new MaybeResponse <ReadJournalResponse>(new ReadJournalResponse(status: result.Status, nextUsn: result.NextUsn)));
}
if (result.Records.Count == 0)
{
return
(new MaybeResponse <ReadJournalResponse>(
new ReadJournalResponse(status: ReadUsnJournalStatus.Success, nextUsn: result.NextUsn)));
}
foreach (var record in result.Records)
{
onUsnRecordReceived(record);
}
Contract.Assume(startUsn < result.NextUsn);
startUsn = result.NextUsn;
if (!endUsn.IsZero)
{
if (startUsn >= endUsn && (--extraReadCount) < 0)
{
return(new MaybeResponse <ReadJournalResponse>(
new ReadJournalResponse(status: ReadUsnJournalStatus.Success, nextUsn: result.NextUsn)));
}
}
}
}
}
}
public void TestEngineStateFileContentTableReuse()
{
SetupHelloWorld();
SetUpConfig();
EngineState lastEngineState = RunEngine("First build");
var firstFCT = lastEngineState.FileContentTable;
XAssert.IsNotNull(firstFCT);
FileIdAndVolumeId inFileIdentity = GetIdentity(GetFullPath(InputFilename));
FileIdAndVolumeId outFileIdentity = GetIdentity(GetFullPath(OutputFilename));
Usn inFileUsn = Usn.Zero;
Usn outFileUsn = Usn.Zero;
ISet <FileIdAndVolumeId> ids = new HashSet <FileIdAndVolumeId>();
XAssert.IsTrue(FileContentTableAccessorFactory.TryCreate(out var accesor, out string error));
firstFCT.VisitKnownFiles(accesor, FileShare.ReadWrite | FileShare.Delete,
(fileIdAndVolumeId, fileHandle, path, knownUsn, knownHash) =>
{
if (fileIdAndVolumeId == inFileIdentity)
{
inFileUsn = knownUsn;
}
else if (fileIdAndVolumeId == outFileIdentity)
{
outFileUsn = knownUsn;
}
ids.Add(fileIdAndVolumeId);
return(true);
});
XAssert.AreNotEqual(Usn.Zero, inFileUsn);
XAssert.AreNotEqual(Usn.Zero, outFileUsn);
// Run engine again
FreshSetUp("change some stuff");
lastEngineState = RunEngine("Second build", engineState: lastEngineState);
var secondFCT = lastEngineState.FileContentTable;
XAssert.AreNotSame(firstFCT, secondFCT); // The FCT gets updated at the end of the run
outFileIdentity = GetIdentity(GetFullPath(OutputFilename)); // Output file changed
bool visitedInput = false;
bool visitedOutput = false;
secondFCT.VisitKnownFiles(accesor, FileShare.ReadWrite | FileShare.Delete,
(fileIdAndVolumeId, fileHandle, path, knownUsn, knownHash) =>
{
if (fileIdAndVolumeId == inFileIdentity)
{
XAssert.IsTrue(ids.Contains(fileIdAndVolumeId));
XAssert.IsTrue(inFileUsn < knownUsn); // We modified the file
inFileUsn = knownUsn;
visitedInput = true;
}
else if (fileIdAndVolumeId == outFileIdentity)
{
XAssert.IsFalse(ids.Contains(fileIdAndVolumeId));
XAssert.IsTrue(outFileUsn < knownUsn); // New output file
outFileUsn = knownUsn;
visitedOutput = true;
}
else
{
XAssert.IsTrue(ids.Contains(fileIdAndVolumeId)); // Other entries are still there
}
return(true);
});
XAssert.IsTrue(visitedInput);
XAssert.IsTrue(visitedOutput);
XAssert.IsTrue(inFileUsn < outFileUsn);
XAssert.AreEqual(firstFCT.Count + 1, secondFCT.Count); // There's a new entry because the new output file has a different fileId
}
/// <summary> /// Creates a new <see cref="VersionedFileIdentity"/> with a specified <see cref="Usn"/>. /// </summary> public VersionedFileIdentity WithUsn(Usn usn) => new VersionedFileIdentity(m_volumeSerialNumber, m_fileId, usn, Kind);
internal static bool TryParse(
ref string line,
out uint processId,
out ReportedFileOperation operation,
out RequestedAccess requestedAccess,
out FileAccessStatus status,
out bool explicitlyReported,
out uint error,
out Usn usn,
out DesiredAccess desiredAccess,
out ShareMode shareMode,
out CreationDisposition creationDisposition,
out FlagsAndAttributes flagsAndAttributes,
out AbsolutePath absolutePath,
out string path,
out string enumeratePattern,
out string processArgs,
out string errorMessage)
{
// TODO: Task 138817: Refactor passing and parsing of report data from native to managed code
operation = ReportedFileOperation.Unknown;
requestedAccess = RequestedAccess.None;
status = FileAccessStatus.None;
processId = error = 0;
usn = default;
explicitlyReported = false;
desiredAccess = 0;
shareMode = ShareMode.FILE_SHARE_NONE;
creationDisposition = 0;
flagsAndAttributes = 0;
absolutePath = AbsolutePath.Invalid;
path = null;
enumeratePattern = null;
processArgs = null;
errorMessage = string.Empty;
var i = line.IndexOf(':');
var index = 0;
if (i > 0)
{
var items = line.Substring(i + 1).Split('|');
if (!Operations.TryGetValue(line.Substring(0, i), out operation))
{
// We could consider the report line malformed in this case; but in practice it is easy to forget to update this parser
// after adding a new call. So let's be conservative about throwing the line out so long as we can parse the important bits to follow.
operation = ReportedFileOperation.Unknown;
}
// When the command line arguments of the process are not reported there will be 12 fields
// When command line arguments are included, everything after the 12th field is the command line argument
// Command line arguments are only reported when the reported file operation is Process
if (operation == ReportedFileOperation.Process)
{
// Make sure the formatting happens only if the condition is false.
if (items.Length < 12)
{
errorMessage = I($"Unexpected message items (potentially due to pipe corruption) for {operation.ToString()} operation. Message '{line}'. Expected >= 12 items, Received {items.Length} items");
return(false);
}
}
else
{
// An ill behaved tool can try to do GetFileAttribute on a file with '|' char. This will result in a failure of the API, but we get a report for the access.
// Allow that by handling such case.
// In Office build there is a call to GetFileAttribute with a small xml document as a file name.
if (items.Length < 12)
{
errorMessage = I($"Unexpected message items (potentially due to pipe corruption) for {operation.ToString()} operation. Message '{line}'. Expected >= 12 items, Received {items.Length} items");
return(false);
}
}
if (
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out processId) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var requestedAccessValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var statusValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var explicitlyReportedValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out error) &&
ulong.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var usnValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var desiredAccessValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var shareModeValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var creationDispositionValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var flagsAndAttributesValue) &&
uint.TryParse(items[index++], NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var absolutePathValue))
{
if (statusValue > (uint)FileAccessStatus.CannotDeterminePolicy)
{
errorMessage = I($"Unknown file access status '{statusValue}'");
return(false);
}
if (requestedAccessValue > (uint)RequestedAccess.All)
{
errorMessage = I($"Unknown requested access '{requestedAccessValue}'");
return(false);
}
requestedAccess = (RequestedAccess)requestedAccessValue;
status = (FileAccessStatus)statusValue;
explicitlyReported = explicitlyReportedValue != 0;
desiredAccess = (DesiredAccess)desiredAccessValue;
shareMode = (ShareMode)shareModeValue;
creationDisposition = (CreationDisposition)creationDispositionValue;
flagsAndAttributes = (FlagsAndAttributes)flagsAndAttributesValue;
absolutePath = new AbsolutePath(unchecked ((int)absolutePathValue));
path = items[index++];
// Detours is only guaranteed to sent at least 12 items, so here (since we are at index 12), we must check if this item is included
enumeratePattern = index < items.Length ? items[index++] : null;
if (requestedAccess != RequestedAccess.Enumerate)
{
// If the requested access is not enumeration, enumeratePattern does not matter.
enumeratePattern = null;
}
if ((operation == ReportedFileOperation.Process) && (items.Length > index))
{
processArgs = items[index++];
while (index < items.Length)
{
processArgs += "|";
processArgs += items[index++];
}
}
else
{
processArgs = string.Empty;
}
usn = new Usn(usnValue);
Contract.Assert(index <= items.Length);
return(true);
}
}
return(false);
}
private bool TryParseAugmentedFileAccess(
ref string data,
out uint processId,
out ReportedFileOperation operation,
out RequestedAccess requestedAccess,
out FileAccessStatus status,
out bool explicitlyReported,
out uint error,
out Usn usn,
out DesiredAccess desiredAccess,
out ShareMode shareMode,
out CreationDisposition creationDisposition,
out FlagsAndAttributes flagsAndAttributes,
out AbsolutePath manifestPath,
out string path,
out string enumeratePattern,
out string processArgs,
out string errorMessage)
{
// An augmented file access has the same structure as a regular one, so let's call
// the usual parser
var result = FileAccessReportLine.TryParse(
ref data,
out processId,
out operation,
out requestedAccess,
out status,
out explicitlyReported,
out error,
out usn,
out desiredAccess,
out shareMode,
out creationDisposition,
out flagsAndAttributes,
out manifestPath,
out path,
out enumeratePattern,
out processArgs,
out errorMessage);
// Augmented file accesses never have the manifest path set, since there is no easy access to the manifest for
// processes to use.
// Let's recreate the manifest path based on the current path and manifest
// The manifest may have its own path table after deserialization, so make sure we use the right one
if (string.IsNullOrEmpty(path) || !AbsolutePath.TryCreate(m_manifest.PathTable, path, out var absolutePath))
{
return(result);
}
var success = m_manifest.TryFindManifestPathFor(absolutePath, out AbsolutePath computedManifestPath, out FileAccessPolicy policy);
// If the manifest specified to not report any accesses, then we just ignore this report line
// We could impose trusted tools the responsibility of knowing this (and not reporting these accesses), but
// this type of coordination is hard to achieve
if ((policy & FileAccessPolicy.ReportAccess) == 0)
{
path = null;
return(true);
}
// If there is no explicit policy for this path, just keep the manifest path as it came from the report
if (!success)
{
return(result);
}
manifestPath = computedManifestPath;
return(result);
}
/// <nodoc />
public FileChangeTrackingRecord(Usn usn, AbsolutePath path, FileChangeTrackingRecord next = null)
{
Usn = usn;
Path = path;
Next = next;
}
public Entry WithNewUsn(Usn usn)
{
return(usn <= Usn ? this : new Entry(usn: usn, hash: Hash, length: Length, timeToLive: TimeToLive));
}
private ReadUsnJournalResult ReadChangesSinceUsn(SafeFileHandle volumeHandle, Usn startUsn)
{
QueryUsnJournalData journalState = QueryJournal(volumeHandle);
// TODO: On a busy volume, we may need to read up to a particular expected USN - or we will fill up the single buffer before finding the expected records.
byte[] buffer = new byte[32768];
ReadUsnJournalResult readJournalResult = FileUtilities.TryReadUsnJournal(volumeHandle, buffer, journalState.UsnJournalId, startUsn: startUsn);
XAssert.AreEqual(ReadUsnJournalStatus.Success, readJournalResult.Status);
return(readJournalResult);
}
