/// <summary>
/// Deserializes into an instance of <see cref="VolumeMap"/>.
/// </summary>
public static VolumeMap Deserialize(BuildXLReader reader)
{
Contract.Requires(reader != null);
var volumeMap = new VolumeMap();
int count = reader.ReadInt32Compact();
for (int i = 0; i < count; ++i)
{
ulong serialNumber = reader.ReadUInt64();
bool isValid = reader.ReadBoolean();
VolumeGuidPath path = isValid ? VolumeGuidPath.Create(reader.ReadString()) : VolumeGuidPath.Invalid;
volumeMap.m_volumePathsBySerial.Add(serialNumber, path);
}
int numJunctionRoots = reader.ReadInt32Compact();
for (int i = 0; i < numJunctionRoots; ++i)
{
string path = reader.ReadString();
var id = FileIdAndVolumeId.Deserialize(reader);
volumeMap.m_junctionRootFileIds.Add(path, id);
}
return(volumeMap);
}
/// <inheritdoc />
public bool TryGetFileHandleAndPathFromFileIdAndVolumeId(FileIdAndVolumeId fileIdAndVolumeId, FileShare fileShare, out SafeFileHandle handle, out string path)
{
path = null;
FileAccessor.OpenFileByIdResult openResult = m_fileAccessor.TryOpenFileById(
fileIdAndVolumeId.VolumeSerialNumber,
fileIdAndVolumeId.FileId,
FileDesiredAccess.GenericRead,
fileShare,
FileFlagsAndAttributes.None,
out handle);
switch (openResult)
{
case FileAccessor.OpenFileByIdResult.FailedToOpenVolume:
case FileAccessor.OpenFileByIdResult.FailedToFindFile:
case FileAccessor.OpenFileByIdResult.FailedToAccessExistentFile:
Contract.Assert(handle == null);
return(false);
case FileAccessor.OpenFileByIdResult.Succeeded:
Contract.Assert(handle != null && !handle.IsInvalid);
path = FileUtilities.GetFinalPathNameByHandle(handle, volumeGuidPath: false);
return(true);
default:
throw Contract.AssertFailure("Unhandled OpenFileByIdResult");
}
}
private static void ReadDictionary(BuildXLReader reader, Dictionary <string, FileIdAndVolumeId> dict)
{
int count = reader.ReadInt32Compact();
for (int i = 0; i < count; ++i)
{
string path = reader.ReadString();
var id = FileIdAndVolumeId.Deserialize(reader);
dict.Add(path, id);
}
}
private void MarkEntryAccessed(FileIdAndVolumeId id, Entry entry)
{
if (entry.TimeToLive == EntryTimeToLive)
{
return; // TTL is already at max; don't bother poking the dictionary.
}
Entry newEntry = entry.WithTimeToLive(EntryTimeToLive);
// We use TryUpdate here since it is possible that a new entry (also with TTL at m_entryTimeToLive)
// was recorded with a new USN. No retries are needed since all changes after load set TTL at m_entryTimeToLive.
Analysis.IgnoreResult(m_entries.TryUpdate(id, newEntry, comparisonValue: entry));
}
/// <nodoc />
public IncorrectFileContentEntry(
string path,
FileIdAndVolumeId fileIdAndVolumeId,
Usn usn,
ContentHash expectedHash,
ContentHash actualHash)
{
Contract.Requires(expectedHash != actualHash);
Contract.Requires(!string.IsNullOrEmpty(path));
Path = path;
FileIdAndVolumeId = fileIdAndVolumeId;
Usn = usn;
ExpectedHash = expectedHash;
ActualHash = actualHash;
}
/// <inheritdoc />
public bool TryGetFileHandleAndPathFromFileIdAndVolumeId(FileIdAndVolumeId fileIdAndVolumeId, FileShare fileShare, out SafeFileHandle handle, out string path)
{
// Accessing hidden files in MacOs.
// Ref: http://www.westwind.com/reference/os-x/invisibles.html
// Another alternative is using fcntl with F_GETPATH that takes a handle and returns the concrete OS path owned by the handle.
path = I($"/.vol/{fileIdAndVolumeId.VolumeSerialNumber}/{fileIdAndVolumeId.FileId.Low}");
OpenFileResult result = FileUtilities.TryCreateOrOpenFile(
path,
FileDesiredAccess.GenericRead,
fileShare,
FileMode.Open,
FileFlagsAndAttributes.None,
out handle);
return(result.Succeeded);
}
public void FileIdAndVolumeIdEquality()
{
var baseValue = new FileIdAndVolumeId(789, new FileId(123, 456));
StructTester.TestEquality(
baseValue: baseValue,
equalValue: baseValue,
notEqualValues: new[]
{
new FileIdAndVolumeId(790, new FileId(123, 456)),
new FileIdAndVolumeId(789, new FileId(124, 456)),
new FileIdAndVolumeId(789, new FileId(123, 457)),
},
eq: (a, b) => a == b,
neq: (a, b) => a != b,
skipHashCodeForNotEqualValues: false);
}
private static LoadResult TryLoadInternal(string fileContentTablePath, byte entryTimeToLive)
{
Contract.Requires(!string.IsNullOrWhiteSpace(fileContentTablePath));
Contract.Requires(entryTimeToLive > 0);
if (!FileUtilities.FileExistsNoFollow(fileContentTablePath))
{
return(LoadResult.FileNotFound(fileContentTablePath));
}
Stopwatch sw = Stopwatch.StartNew();
try
{
using (FileStream stream = FileUtilities.CreateFileStream(
fileContentTablePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete,
// Ok to evict the file from standby since the file will be overwritten and never reread from disk after this point.
FileOptions.SequentialScan))
{
try
{
Analysis.IgnoreResult(s_fileEnvelope.ReadHeader(stream));
}
catch (BuildXLException ex)
{
return(LoadResult.InvalidFormat(fileContentTablePath, ex.LogEventMessage, sw.ElapsedMilliseconds));
}
using (var reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
var loadedTable = new FileContentTable();
uint numberOfEntries = reader.ReadUInt32();
int hashLength = ContentHashingUtilities.HashInfo.ByteLength;
var hashBuffer = new byte[hashLength];
for (uint i = 0; i < numberOfEntries; i++)
{
// Key: Volume and file ID
var fileIdAndVolumeId = FileIdAndVolumeId.Deserialize(reader);
// Entry: USN, hash, length, time to live.
Usn usn = new Usn(reader.ReadUInt64());
int hashBytesRead = 0;
while (hashBytesRead != hashLength)
{
int thisRead = reader.Read(hashBuffer, hashBytesRead, hashLength - hashBytesRead);
if (thisRead == 0)
{
return(LoadResult.InvalidFormat(fileContentTablePath, "Unexpected end of stream", sw.ElapsedMilliseconds));
}
hashBytesRead += thisRead;
Contract.Assert(hashBytesRead <= hashLength);
}
long length = reader.ReadInt64();
byte thisEntryTimeToLive = reader.ReadByte();
if (thisEntryTimeToLive == 0)
{
return(LoadResult.InvalidFormat(fileContentTablePath, "TTL value must be positive", sw.ElapsedMilliseconds));
}
thisEntryTimeToLive = Math.Min(thisEntryTimeToLive, entryTimeToLive);
Contract.Assert(thisEntryTimeToLive > 0);
// We've loaded this entry just now and so clearly haven't used it yet. Tentatively decrement the TTL
// for the in-memory table; if the table is saved again without using this entry, the TTL will stay at this
// lower value.
thisEntryTimeToLive--;
var observedVersionAndHash = new Entry(usn, ContentHashingUtilities.CreateFrom(hashBuffer), length, thisEntryTimeToLive);
bool added = loadedTable.m_entries.TryAdd(fileIdAndVolumeId, observedVersionAndHash);
Contract.Assume(added);
}
loadedTable.Counters.AddToCounter(FileContentTableCounters.NumEntries, loadedTable.Count);
return(LoadResult.Success(fileContentTablePath, loadedTable, sw.ElapsedMilliseconds));
}
}
}
catch (Exception ex)
{
return(LoadResult.Exception(fileContentTablePath, ex, sw.ElapsedMilliseconds));
}
}
/// <summary>
/// Records a <see cref="ContentHash" /> for the given file handle. This hash mapping will be persisted to disk if the
/// table is saved with <see cref="SaveAsync" />. The given file handle should be opened with at most Read sharing
/// (having the handle should ensure the file is not being written).
/// This returns a <see cref="VersionedFileIdentityAndContentInfo"/>:
/// - The identity has the kind <see cref="VersionedFileIdentity.IdentityKind.StrongUsn"/> if a USN-based identity was successfully established;
/// the identity may have kind <see cref="VersionedFileIdentity.IdentityKind.Anonymous"/> if such an identity was unavailable.
/// - Regardless, the contained <see cref="FileContentInfo"/> contains the actual length of the stream corresponding to <paramref name="hash"/>.
/// </summary>
/// <remarks>
/// An overload taking a file path is intentionally not provided. This should be called after hashing or writing a file,
/// but before closing the handle. This way, there is no race between establishing the file's hash, some unrelated writer,
/// and recording its file version (e.g., USN) to hash mapping.
/// Note that this results in a small amount of I/O (e.g., on Windows, a file open and USN query), but never hashes the file or reads its contents.
/// The <paramref name="strict"/> corresponds to the <c>flush</c> parameter of <see cref="VersionedFileIdentity.TryEstablishStrong"/>
/// </remarks>
public VersionedFileIdentity RecordContentHash(
string path,
SafeFileHandle handle,
ContentHash hash,
long length,
bool?strict = default)
{
Contract.Requires(handle != null);
Contract.Requires(!string.IsNullOrWhiteSpace(path));
using (Counters.StartStopwatch(FileContentTableCounters.RecordContentHashDuration))
{
// TODO: The contract below looks very nice but breaks tons of UT
// Fix the tests and enable the contract.
// Contract.Requires(FileContentInfo.IsValidLength(length, hash));
// Here we write a new change journal record for this file to get a 'strong' identity. This means that the USN -> hash table
// only ever contains USNs whose records have the 'close' reason set. Recording USNs without that
// reason set would not be correct; it would be possible that multiple separate changes (e.g. writes)
// were represented with the same USN, and so intermediate USNs do not necessarily correspond to exactly
// one snapshot of a file. See http://msdn.microsoft.com/en-us/library/windows/desktop/aa363803(v=vs.85).aspx
Possible <VersionedFileIdentity, Failure <VersionedFileIdentity.IdentityUnavailabilityReason> > possibleVersionedIdentity =
TryEstablishStrongIdentity(handle, flush: strict == true);
if (!possibleVersionedIdentity.Succeeded)
{
if (Interlocked.CompareExchange(ref m_changeJournalWarningLogged, 1, 0) == 0)
{
Tracing.Logger.Log.StorageFileContentTableIgnoringFileSinceVersionedFileIdentityIsNotSupported(
Events.StaticContext,
path,
possibleVersionedIdentity.Failure.DescribeIncludingInnerFailures());
}
return(VersionedFileIdentity.Anonymous);
}
VersionedFileIdentity identity = possibleVersionedIdentity.Result;
var newEntry = new Entry(identity.Usn, hash, length, EntryTimeToLive);
// We allow concurrent update attempts with different observed USNs.
// This is useful and relevant for two reasons:
// - Querying a 'strong' identity (TryEstablishStrongIdentity) generates a new CLOSE record every time.
// - Creating hardlinks generates 'hardlink change' records.
// So, concurrently creating and recording (or even just recording) different links is possible, and
// keeping the last stored entry (rather than highest-USN entry) can introduce false positives.
var fileIdAndVolumeId = new FileIdAndVolumeId(identity.VolumeSerialNumber, identity.FileId);
m_entries.AddOrUpdate(
new FileIdAndVolumeId(identity.VolumeSerialNumber, identity.FileId),
newEntry,
updateValueFactory: (key, existingEntry) =>
{
if (existingEntry.Usn > newEntry.Usn)
{
return(existingEntry);
}
if (newEntry.Hash == existingEntry.Hash)
{
Counters.IncrementCounter(FileContentTableCounters.NumUsnMismatch);
Tracing.Logger.Log.StorageUsnMismatchButContentMatch(
Events.StaticContext,
path,
existingEntry.Usn.Value,
newEntry.Usn.Value,
existingEntry.Hash.ToHex());
}
else
{
// Stale USN.
Counters.IncrementCounter(FileContentTableCounters.NumContentMismatch);
}
return(newEntry);
});
Tracing.Logger.Log.StorageRecordNewKnownUsn(
Events.StaticContext,
path,
identity.FileId.High,
identity.FileId.Low,
identity.VolumeSerialNumber,
identity.Usn.Value,
hash.ToHex());
return(identity);
}
}
/// <summary>
/// Retrieves an already-known <see cref="ContentHash" /> for the given file handle. If no such hash is available (such as
/// if the file has been modified since a hash was last recorded), null is returned instead.
/// </summary>
/// <remarks>
/// Note that this results in a small amount of I/O (e.g., on Windows, a file open and USN query), but never hashes the file or reads its contents.
/// </remarks>
public VersionedFileIdentityAndContentInfo?TryGetKnownContentHash(string path, SafeFileHandle handle)
{
Contract.Requires(!string.IsNullOrWhiteSpace(path));
Contract.Requires(handle != null);
using (Counters.StartStopwatch(FileContentTableCounters.GetContentHashDuration))
{
Possible <VersionedFileIdentity, Failure <VersionedFileIdentity.IdentityUnavailabilityReason> > possibleVersionedIdentity =
TryQueryWeakIdentity(handle);
if (!possibleVersionedIdentity.Succeeded)
{
// We fail quietly for disabled journals on the query side; instead attempting to record a hash will fail.
Contract.Assume(
possibleVersionedIdentity.Failure.Content == VersionedFileIdentity.IdentityUnavailabilityReason.NotSupported);
Tracing.Logger.Log.StorageVersionedFileIdentityNotSupportedMiss(Events.StaticContext, path);
return(null);
}
VersionedFileIdentity identity = possibleVersionedIdentity.Result;
var fileIdInfo = new FileIdAndVolumeId(identity.VolumeSerialNumber, identity.FileId);
// We have a valid identity, but that identity is 'weak' and may correspond to an intermediate record (one without 'close' set).
// We cannot discard such records here since we can't obtain a real 'Reason' field for a file's current USN record.
// But we do know that any intermediate record will be a miss below, since we only record 'close' records (strong identities)
// (see RecordContentHashAsync).
Entry knownEntry;
bool foundEntry = m_entries.TryGetValue(fileIdInfo, out knownEntry);
if (!foundEntry)
{
Counters.IncrementCounter(FileContentTableCounters.NumFileIdMismatch);
Tracing.Logger.Log.StorageUnknownFileMiss(
Events.StaticContext,
path,
identity.FileId.High,
identity.FileId.Low,
identity.VolumeSerialNumber,
identity.Usn.Value);
return(null);
}
var staleUsn = identity.Usn != knownEntry.Usn;
if (staleUsn)
{
Tracing.Logger.Log.StorageUnknownUsnMiss(
Events.StaticContext,
path,
identity.FileId.High,
identity.FileId.Low,
identity.VolumeSerialNumber,
readUsn: identity.Usn.Value,
knownUsn: knownEntry.Usn.Value,
knownContentHash: knownEntry.Hash.ToHex());
return(null);
}
MarkEntryAccessed(fileIdInfo, knownEntry);
Counters.IncrementCounter(FileContentTableCounters.NumHit);
Tracing.Logger.Log.StorageKnownUsnHit(
Events.StaticContext,
path,
identity.FileId.High,
identity.FileId.Low,
identity.VolumeSerialNumber,
usn: knownEntry.Usn.Value,
contentHash: knownEntry.Hash.ToHex());
// Note that we return a 'strong' version of the weak identity; since we matched an entry in the table, we know that the USN
// actually corresponds to a strong identity (see RecordContentHashAsync).
return(new VersionedFileIdentityAndContentInfo(
new VersionedFileIdentity(
identity.VolumeSerialNumber,
identity.FileId,
identity.Usn,
VersionedFileIdentity.IdentityKind.StrongUsn),
new FileContentInfo(knownEntry.Hash, knownEntry.Length)));
}
}
/// <summary>
/// Creates an instance of <see cref="ChangedFileIdInfo" />.
/// </summary>
public ChangedFileIdInfo(FileIdAndVolumeId fileIdAndVolumeId, UsnRecord usnRecord)
{
FileIdAndVolumeId = fileIdAndVolumeId;
UsnRecord = usnRecord;
}
public void TestEngineStateFileContentTableReuse()
{
SetupHelloWorld();
SetUpConfig();
EngineState lastEngineState = RunEngine("First build");
var firstFCT = lastEngineState.FileContentTable;
XAssert.IsNotNull(firstFCT);
FileIdAndVolumeId inFileIdentity = GetIdentity(GetFullPath(InputFilename));
FileIdAndVolumeId outFileIdentity = GetIdentity(GetFullPath(OutputFilename));
Usn inFileUsn = Usn.Zero;
Usn outFileUsn = Usn.Zero;
ISet <FileIdAndVolumeId> ids = new HashSet <FileIdAndVolumeId>();
XAssert.IsTrue(FileContentTableAccessorFactory.TryCreate(out var accesor, out string error));
firstFCT.VisitKnownFiles(accesor, FileShare.ReadWrite | FileShare.Delete,
(fileIdAndVolumeId, fileHandle, path, knownUsn, knownHash) =>
{
if (fileIdAndVolumeId == inFileIdentity)
{
inFileUsn = knownUsn;
}
else if (fileIdAndVolumeId == outFileIdentity)
{
outFileUsn = knownUsn;
}
ids.Add(fileIdAndVolumeId);
return(true);
});
XAssert.AreNotEqual(Usn.Zero, inFileUsn);
XAssert.AreNotEqual(Usn.Zero, outFileUsn);
// Run engine again
FreshSetUp("change some stuff");
lastEngineState = RunEngine("Second build", engineState: lastEngineState);
var secondFCT = lastEngineState.FileContentTable;
XAssert.AreNotSame(firstFCT, secondFCT); // The FCT gets updated at the end of the run
outFileIdentity = GetIdentity(GetFullPath(OutputFilename)); // Output file changed
bool visitedInput = false;
bool visitedOutput = false;
secondFCT.VisitKnownFiles(accesor, FileShare.ReadWrite | FileShare.Delete,
(fileIdAndVolumeId, fileHandle, path, knownUsn, knownHash) =>
{
if (fileIdAndVolumeId == inFileIdentity)
{
XAssert.IsTrue(ids.Contains(fileIdAndVolumeId));
XAssert.IsTrue(inFileUsn < knownUsn); // We modified the file
inFileUsn = knownUsn;
visitedInput = true;
}
else if (fileIdAndVolumeId == outFileIdentity)
{
XAssert.IsFalse(ids.Contains(fileIdAndVolumeId));
XAssert.IsTrue(outFileUsn < knownUsn); // New output file
outFileUsn = knownUsn;
visitedOutput = true;
}
else
{
XAssert.IsTrue(ids.Contains(fileIdAndVolumeId)); // Other entries are still there
}
return(true);
});
XAssert.IsTrue(visitedInput);
XAssert.IsTrue(visitedOutput);
XAssert.IsTrue(inFileUsn < outFileUsn);
XAssert.AreEqual(firstFCT.Count + 1, secondFCT.Count); // There's a new entry because the new output file has a different fileId
}
/// <summary>
/// Creates an instance of <see cref="ChangedFileIdInfo" />.
/// </summary>
public ChangedFileIdInfo(FileIdAndVolumeId fileIdAndVolumeId, UsnRecord usnRecord, Usn?lastTrackedUsn = default)
{
FileIdAndVolumeId = fileIdAndVolumeId;
UsnRecord = usnRecord;
LastTrackedUsn = lastTrackedUsn;
}
