public Possible <FileChangeTrackingSet.EnumerationResult> TryEnumerateDirectoryAndTrackMembership(
string path,
Action <string, FileAttributes> handleEntry)
{
Contract.Requires(path != null);
Contract.Requires(handleEntry != null);
var possibleFingerprintResult = DirectoryMembershipTrackingFingerprinter.ComputeFingerprint(path, handleEntry);
if (!possibleFingerprintResult.Succeeded)
{
return(possibleFingerprintResult.Failure);
}
PathExistence existence = possibleFingerprintResult.Result.PathExistence;
Possible <FileChangeTrackingSet.ConditionalTrackingResult> trackingResult = new Failure <string>("Tracking membership failed");
lock (m_lock)
{
if (existence == PathExistence.ExistsAsDirectory)
{
trackingResult = FileChangeTrackingSet.ConditionalTrackingResult.Tracked;
PathsWithTrackedMembership.Add(Path.GetFullPath(path));
}
PathsWithTrackedExistence.Add(Path.GetFullPath(path));
}
return(new FileChangeTrackingSet.EnumerationResult(
possibleFingerprintResult.Result.Fingerprint,
possibleFingerprintResult.Result.PathExistence,
trackingResult));
}
/// <summary>
/// Creates an instance of <see cref="DirectoryMembershipFingerprintResult"/>.
/// </summary>
public DirectoryMembershipFingerprintResult(
DirectoryMembershipTrackingFingerprint fingerprint,
PathExistence pathExistence,
int memberCount)
{
Fingerprint = fingerprint;
PathExistence = pathExistence;
MemberCount = memberCount;
}
public void VerifyExistence(string fullPath, FileSystemViewMode mode, PathExistence existence, bool shouldBeProbed)
{
ProbePaths.Clear();
var path = GetPath(fullPath);
var actualExistence = FileSystemView.GetExistence(path, mode);
Assert.Equal(existence, actualExistence.Result);
Assert.Equal(shouldBeProbed, ProbePaths.Contains(path));
}
public static ProvingContext ToProvingContext(this Application application)
{
var paths = new PathExistence(
(from @this in application.PropertiesOf <Paths>() select @this.Content).SelectMany(p => p));
var hosts = new HostAvailability(
(from @this in application.PropertiesOf <Hosts>() select @this.Content).SelectMany(p => p));
return(new ProvingContext(paths, hosts));
}
public async void Should_not_prove_directory_existence_if_does_not_exists()
{
const string fakePath = "/var/tests";
var mock = ArbitraryIO.Directories(ArbitraryIO.DirectoryPaths(samples: Maybe.Nothing <ushort>()));
var sut = new PathExistence(mock, new Paths().AddDirectory(fakePath).Content);
var outcome = await sut.VerifyAsync();
outcome.Should().NotBeNull()
.And.BeOfType <Bad <Evidence, string> >()
.Which.FailedWith().Single().Should().Be($"{fakePath} directory is not found.");
}
public Possible <PathExistence> TryProbeAndTrackPath(string path, bool?isReadOnly = default)
{
Contract.Requires(path != null);
PathExistence existence = GetExistence(path);
lock (m_lock)
{
PathsWithTrackedExistence.Add(Path.GetFullPath(path));
}
return(existence);
}
public async void Should_prove_directory_existence_if_exists()
{
const string fakePath = "/var/tests";
var mock = ArbitraryIO.Directories(ArbitraryIO.DirectoryPaths(
samples: Maybe.Nothing <ushort>(), fakePath));
var sut = new PathExistence(mock, new Paths().AddDirectory(fakePath).Content);
var outcome = await sut.VerifyAsync();
outcome.Should().NotBeNull()
.And.BeOfType <Ok <Evidence, string> >()
.Which.SucceededWith().Should().BeEquivalentTo(new Evidence(typeof(PathExistence),
new Paths().AddDirectory(fakePath).Content));
}
public void ThrowExceptionIfColumnNumberIsOutOfRange(int expectedColumnNumber)
{
const string csvPath = @"D:\a\review-case-s21b11\review-case-s21b11\sample-review\review-report.csv";
var checkPath = PathExistence.CheckFileExistOrNot(csvPath);
if (checkPath)
{
var lines = System.IO.File.ReadAllLines(csvPath);
Assert.Throws <System.IndexOutOfRangeException>(() => ColumnFilter.PrintCsvColumn(lines, expectedColumnNumber));
}
else
{
Assert.True(false);
}
}
public async void Should_prove_files_and_dirs_existence_if_exist()
{
const string fakeFilePath = "/var/tests/fake.txt";
const string fakeDirectoryPath = "/var/tests";
var mock = ArbitraryIO.Files(ArbitraryIO.FilePaths(
samples: Maybe.Nothing <ushort>(), fakeFilePath));
ArbitraryIO.AddDirectories((MockFileSystem)mock, fakeDirectoryPath);
var paths = new Paths().AddFile(fakeFilePath).AddDirectory(fakeDirectoryPath).Content;
var sut = new PathExistence(mock, paths);
var outcome = await sut.VerifyAsync();
outcome.Should().NotBeNull()
.And.BeOfType <Ok <Evidence, string> >()
.Which.SucceededWith().Should().BeEquivalentTo(new Evidence(typeof(PathExistence),
paths));
}
/// <summary>
/// Checks if the given path is an output under a shared opaque by verifying whether <see cref="WellKnownTimestamps.OutputInSharedOpaqueTimestamp"/> is the creation time of the file
/// </summary>
/// <remarks>
/// If the given path is a directory, it is always considered part of a shared opaque
/// </remarks>
public static bool IsSharedOpaqueOutput(string expandedPath, out PathExistence pathExistence)
{
// On Windows: FOLLOW symlinks
// - directory symlinks are not fully supported (e.g., producing directory symlinks)
// - other reparse points are at play (e.g., junctions, Helium containers) which necessitate
// that we transparently follow those reparse points
// On non-Windows: NO_FOLLOW symlinks
// - directory symlinks are supported
// - all symlinks are treated uniformly as files
// - if 'expandedPath' is a symlink pointing to a non-existent file, 'expandedPath' should hence
// still be treated as an existent file
// - similarly, if 'expandedPath' is a symlink pointing to a directory, it should still be treated as a file
bool followSymlink = !OperatingSystemHelper.IsUnixOS;
// If the file is not there (the check may be happening against a reported write that later got deleted)
// then there is nothing to do
var maybeResult = FileUtilities.TryProbePathExistence(expandedPath, followSymlink);
if (maybeResult.Succeeded && maybeResult.Result == PathExistence.Nonexistent)
{
pathExistence = PathExistence.Nonexistent;
return(true);
}
// We don't really track directories as part of shared opaques.
// So we consider them all potential members and return true.
// It is important to track directory symlinks, because they are considered files for sake of shared opaque scrubbing.
if (maybeResult.Succeeded && maybeResult.Result == PathExistence.ExistsAsDirectory)
{
pathExistence = PathExistence.ExistsAsDirectory;
return(true);
}
pathExistence = PathExistence.ExistsAsFile;
return(OperatingSystemHelper.IsMacOS
? XattrBased.IsSharedOpaqueOutput(expandedPath)
: TimestampBased.IsSharedOpaqueOutput(expandedPath));
}
public void AddRealPath(string fullPath, PathExistence existence)
{
if (existence == PathExistence.Nonexistent)
{
return;
}
var path = AbsolutePath.Create(m_pathTable, fullPath);
FileOrDirectoryArtifact member = FileOrDirectoryArtifact.Invalid;
while (path.IsValid)
{
if (existence == PathExistence.ExistsAsFile)
{
m_files.Add(path);
member = new FileArtifact(path);
}
else
{
var members = m_directories.GetOrAdd(path, p => new HashSet <FileOrDirectoryArtifact>());
if (member.IsValid)
{
if (!members.Add(member))
{
return;
}
}
member = DirectoryArtifact.CreateWithZeroPartialSealId(path);
}
existence = PathExistence.ExistsAsDirectory;
path = path.GetParent(m_pathTable);
}
}
public Possible <Unit, Failure> TryDelete()
{
Contract.Requires(IsValid);
var maybeExistence = FileUtilities.TryProbePathExistence(Path, followSymlink: false);
PathExistence pathExistence = PathExistence.Nonexistent;
if (maybeExistence.Succeeded)
{
pathExistence = maybeExistence.Result;
}
else
{
if (File.Exists(Path))
{
pathExistence = PathExistence.ExistsAsFile;
}
else if (Directory.Exists(Path))
{
pathExistence = PathExistence.ExistsAsDirectory;
}
BuildXL.Storage.Tracing.Logger.Log.FileMaterializationMismatchFileExistenceResult(
Events.StaticContext,
Path,
maybeExistence.Failure.Describe(),
pathExistence.ToString());
}
if (pathExistence == PathExistence.ExistsAsFile)
{
m_fileInfoPriorDeletion = new FileInfo(Path);
}
m_deletionTime = DateTime.UtcNow;
switch (pathExistence)
{
case PathExistence.ExistsAsFile:
{
var deleteFileResult = FileUtilities.TryDeleteFile(Path);
if (!deleteFileResult.Succeeded)
{
return(deleteFileResult.Failure.Annotate(I($"Failed to delete file '{Path}'")));
}
break;
}
case PathExistence.ExistsAsDirectory:
{
FileUtilities.DeleteDirectoryContents(Path, deleteRootDirectory: true);
break;
}
default:
break;
}
return(Unit.Void);
}
public void TestingForTheNonExistenceOfCsvFile(string csvPath)
{
//const string csvPath = @"E:\Philips\Review-case-s21b11-master\sample-review\Review-report.csv";
PathExistence.CheckFileExistOrNot(csvPath);
Assert.False(PathExistence.CheckFileExistOrNot(csvPath));
}
public void TestingForTheExistenceOfCsvFile()
{
const string csvPath = @"D:\a\review-case-s21b11\review-case-s21b11\sample-review\review-report.csv";
Assert.True(PathExistence.CheckFileExistOrNot(csvPath));
}
public void VerifyGraphExistence(string fullPath, FileSystemViewMode mode, PathExistence existence)
{
Contract.Assert(mode != FileSystemViewMode.Real);
// Graph queries should never probe the file system
VerifyExistence(fullPath, mode, existence, shouldBeProbed: false);
var path = GetPath(fullPath);
if (existence == PathExistence.ExistsAsFile)
{
// File should be queried from underlying pip graph file system
Assert.True(TryGetLatestFileArtifactPaths.Contains(path) && m_latestWriteCountByPath.ContainsKey(path));
}
else
{
// Directory and non-existent paths will query underlying pip graph file system but will not be found
Assert.True(TryGetLatestFileArtifactPaths.Contains(path));
if (mode == FileSystemViewMode.FullGraph)
{
Assert.False(m_latestWriteCountByPath.ContainsKey(path));
}
else
{
Assert.True(!m_latestWriteCountByPath.ContainsKey(path) || m_latestWriteCountByPath[path] == 0);
}
}
}
/// <summary>
/// Adds the existence state of the path if not present or returns the current cached existence state of the path
/// </summary>
/// <remarks>
/// If flags are provided, those are added to potentially existing flags
/// </remarks>
private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, out bool added, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None)
{
PathExistence result = existence;
var originalPath = path;
while (path.IsValid)
{
var getOrAddResult = PathExistenceCache.GetOrAdd(path, FileSystemEntry.Create(mode, existence).SetFlag(flags));
if (getOrAddResult.IsFound)
{
var currentExistence = getOrAddResult.Item.Value.GetExistence(mode);
var currentFlags = getOrAddResult.Item.Value.Flags;
if (currentExistence != null && currentFlags.HasFlag(flags))
{
if (originalPath != path)
{
// Parent entry with existence already exists, just return the result
added = true;
return(result);
}
else
{
// Entry with existence already exists, just return the value
added = false;
return(currentExistence.Value);
}
}
// found entry for the 'path', but it does not contain existence for the specified 'mode'
var updateResult = PathExistenceCache.AddOrUpdate(path, (mode, existence),
(key, data) => FileSystemEntry.Create(data.mode, data.existence).SetFlag(flags),
(key, data, oldValue) => oldValue.TryUpdate(data.mode, data.existence).SetFlag(flags));
// the same entry might be updated concurrently; check whether we lost the race, and if we did, stop further processing
currentExistence = updateResult.OldItem.Value.GetExistence(mode);
currentFlags = updateResult.OldItem.Value.Flags;
if (currentExistence != null && currentFlags.HasFlag(flags))
{
if (originalPath != path)
{
// Parent entry with existence already exists, just return the result
added = true;
return(result);
}
else
{
// Entry with existence already exists, just return the value
added = false;
return(currentExistence.Value);
}
}
}
// Only register parents as directories if path exists AND updateParents=true
if (existence == PathExistence.Nonexistent || !updateParents)
{
break;
}
// Set ancestor paths existence to directory existent for existent paths
existence = PathExistence.ExistsAsDirectory;
path = path.GetParent(PathTable);
}
added = true;
return(result);
}
/// <summary>
/// Adds the existence state of the path if not present or returns the current cached existence state of the path
/// </summary>
private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true, FileSystemEntryFlags flags = FileSystemEntryFlags.None)
{
return(GetOrAddExistence(path, mode, existence, out var added, updateParents, flags));
}
/// <summary>
/// Gets the existence of a path for the given file system view mode if set
/// </summary>
private bool TryGetKnownPathExistence(AbsolutePath path, FileSystemViewMode mode, out PathExistence existence)
{
if (mode == FileSystemViewMode.FullGraph || mode == FileSystemViewMode.Output)
{
if (ExistsInGraphFileSystem(PipGraph.TryGetLatestFileArtifactForPath(path), mode))
{
existence = PathExistence.ExistsAsFile;
return(true);
}
}
// For graph file systems, we query the path existence cache for existence of directories. Files are tracked
// by the pip graph. For dynamic files, we need to check PathExistenceCache.
existence = PathExistence.Nonexistent;
FileSystemEntry entry;
return(PathExistenceCache.TryGetValue(path, out entry) && entry.TryGetExistence(mode, out existence));
}
/// <summary>
/// Adds the existence state of the path if not present or returns the current cached existence state of the path
/// </summary>
private PathExistence GetOrAddExistence(AbsolutePath path, FileSystemViewMode mode, PathExistence existence, bool updateParents = true)
{
return(GetOrAddExistence(path, mode, existence, out var added, updateParents));
}
private void AssertPathExistence(PathExistence expected, Possible <PathExistence, NativeFailure> maybeFileExistence)
{
XAssert.IsTrue(maybeFileExistence.Succeeded);
XAssert.AreEqual(expected, maybeFileExistence.Result);
}
