/// <nodoc />
public DistributedCentralStoreConfiguration(AbsolutePath cacheRoot)
{
CacheRoot = cacheRoot;
}
private static void AssertHasExclusiveAccessToFile([NotNull] BaseEntry entry, [NotNull] AbsolutePath absolutePath)
{
if (entry is FileEntry file && file.IsOpen())
{
throw ErrorFactory.System.FileIsInUse(absolutePath.GetText());
}
}
/// <summary> /// <see cref="TryGetFileAsync"/> /// </summary> protected abstract Task <BoolResult> TryGetFileCoreAsync(OperationContext context, string storageId, AbsolutePath targetFilePath, bool isImmutable);
protected override async Task <Result <string> > UploadFileCoreAsync(OperationContext context, AbsolutePath file, string blobName, bool garbageCollect = false)
{
// Add the file to CAS and register with global content location store
var hashTask = PutAndRegisterFileAsync(context, file, hash: null);
// Upload to fallback storage so file is available if needed from there
var innerStorageIdTask = _fallbackStorage.UploadFileAsync(context, file, blobName, garbageCollect).ThrowIfFailureAsync();
await Task.WhenAll(hashTask, innerStorageIdTask);
var hash = await hashTask;
var innerStorageId = await innerStorageIdTask;
return(CreateCompositeStorageId(hash, innerStorageId));
}
private async Task <ContentHashWithSize> TryGetFromFallbackAndPutAsync(OperationContext context, AbsolutePath targetFilePath, string fallbackStorageId)
{
// In the success case the content will be put at targetFilePath
await _fallbackStorage.TryGetFileAsync(context, fallbackStorageId, targetFilePath).ThrowIfFailure();
var putResult = await _privateCas.PutFileAsync(context, targetFilePath, FileRealizationMode.Copy, _hashType, pinRequest : null).ThrowIfFailure();
return(new ContentHashWithSize(putResult.ContentHash, putResult.ContentSize));
}
public override void PutFileStop(Context context, PutResult result, bool trusted, AbsolutePath path, FileRealizationMode mode, Severity successSeverity)
{
if (_eventSource.IsEnabled())
{
_eventSource.PutFileStop(
context.Id.ToString(), result.Succeeded, result.ErrorMessage, result.ContentHash.ToString());
}
base.PutFileStop(context, result, trusted, path, mode, successSeverity: DiagnosticLevelSeverity(result.Duration));
}
protected override async Task <BoolResult> TouchBlobCoreAsync(OperationContext context, AbsolutePath file, string storageId, bool isUploader)
{
var(hash, fallbackStorageId) = ParseCompositeStorageId(storageId);
// Need to touch in fallback storage as well so it knows the content is still in use
var touchTask = _fallbackStorage.TouchBlobAsync(context, file, fallbackStorageId, isUploader).ThrowIfFailure();
// Ensure content is present in private CAS and registered
var registerTask = PutAndRegisterFileAsync(context, file, hash);
await Task.WhenAll(touchTask, registerTask);
return(BoolResult.Success);
}
/// <summary>
/// Creates a token from the information with the given path
/// </summary>
public TokenData ToToken(AbsolutePath path)
{
return(IsValid ? new TokenData(path, Line, Position, Text) : TokenData.Invalid);
}
protected abstract IContentStore CreateStore(AbsolutePath rootPath, ContentStoreConfiguration configuration);
private Possible <Unit> TryReportInputs(List <FileArtifactKeyedHash> hashes)
{
var dynamicDirectoryMap = new Dictionary <DirectoryArtifact, List <FileArtifact> >();
var failedFiles = new List <(FileArtifact file, ContentHash hash)>();
var fileContentManager = m_environment.State.FileContentManager;
foreach (FileArtifactKeyedHash fileArtifactKeyedHash in hashes)
{
FileArtifact file;
if (fileArtifactKeyedHash.AssociatedDirectories != null && fileArtifactKeyedHash.AssociatedDirectories.Count != 0)
{
// All workers have the same entries in the path table up to the schedule phase. Dynamic outputs add entries to the path table of the worker that generates them.
// Dynamic outputs can be generated by one worker, but consumed by another, which potentially is not the master. Thus, the two workers do not have the same entries
// in the path table.
// The approach we have here may be sub-optimal(the worker may have had the paths already). But it ensures correctness.
// Need to create absolute path because the path is potentially not in the path table.
file = new FileArtifact(
AbsolutePath.Create(m_environment.Context.PathTable, fileArtifactKeyedHash.PathString),
fileArtifactKeyedHash.RewriteCount);
foreach (var bondDirectoryArtifact in fileArtifactKeyedHash.AssociatedDirectories)
{
var directory = new DirectoryArtifact(
new AbsolutePath(bondDirectoryArtifact.DirectoryPathValue),
bondDirectoryArtifact.DirectorySealId,
bondDirectoryArtifact.IsDirectorySharedOpaque);
if (!dynamicDirectoryMap.TryGetValue(directory, out var files))
{
files = new List <FileArtifact>();
dynamicDirectoryMap.Add(directory, files);
}
files.Add(file);
}
}
else
{
file = fileArtifactKeyedHash.File;
}
if (fileArtifactKeyedHash.IsSourceAffected)
{
fileContentManager.SourceChangeAffectedInputs.ReportSourceChangedAffectedFile(file.Path);
}
var materializationInfo = fileArtifactKeyedHash.GetFileMaterializationInfo(m_environment.Context.PathTable);
if (!fileContentManager.ReportWorkerPipInputContent(
m_appLoggingContext,
file,
materializationInfo))
{
failedFiles.Add((file, materializationInfo.Hash));
}
}
foreach (var directoryAndContents in dynamicDirectoryMap)
{
fileContentManager.ReportDynamicDirectoryContents(directoryAndContents.Key, directoryAndContents.Value, PipOutputOrigin.NotMaterialized);
}
if (failedFiles.Count != 0)
{
return(new ArtifactMaterializationFailure(failedFiles.ToReadOnlyArray(), m_environment.Context.PathTable));
}
return(Unit.Void);
}
private AbsolutePath CreateAbsolutePath(string path)
{
return(AbsolutePath.Create(PathTable, path));
}
/// <summary>
/// Converts this LineInfo to a LocationData with the given path.
/// </summary>
public LocationData ToLocationData(AbsolutePath path)
{
return(!path.IsValid ? LocationData.Invalid : new LocationData(path, Line, Position));
}
/// <inheritdoc /> public CheckpointConfiguration(AbsolutePath workingDirectory) => WorkingDirectory = workingDirectory;
/// <nodoc />
public LocalDiskCentralStoreConfiguration(AbsolutePath workingDirectory, string checkpointsKey)
: base(checkpointsKey) => WorkingDirectory = workingDirectory;
protected void RestartEngine()
{
var pathTable = new PathTable();
FileSystem = new PassThroughMutableFileSystem(pathTable);
Context = EngineContext.CreateNew(CancellationToken.None, pathTable, FileSystem);
MainSourceResolverModules = new List <AbsolutePath>();
var rootPath = AbsolutePath.Create(Context.PathTable, TestRoot);
var logsPath = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "logs");
Configuration = new CommandLineConfiguration()
{
DisableDefaultSourceResolver = true,
Resolvers = new List <IResolverSettings>
{
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = MainSourceResolverModules,
},
new SourceResolverSettings
{
Kind = "SourceResolver",
Modules = new List <AbsolutePath>
{
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Prelude", "package.config.dsc")),
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Transformers", "package.config.dsc")),
AbsolutePath.Create(Context.PathTable, Path.Combine(GetTestExecutionLocation(), "Sdk", "Deployment", "module.config.dsc")),
},
},
},
Layout =
{
SourceDirectory = rootPath,
OutputDirectory = Combine(rootPath, "out"),
ObjectDirectory = Combine(rootPath, "obj"),
CacheDirectory = Combine(AbsolutePath.Create(Context.PathTable, TemporaryDirectory), "cache"),
},
Cache =
{
CacheSpecs = SpecCachingOption.Disabled,
CacheLogFilePath = logsPath.Combine(Context.PathTable,PathAtom.Create(Context.StringTable, "cache.log")),
},
Engine =
{
ReuseEngineState = false,
LogStatistics = false,
TrackBuildsInUserFolder = false,
},
FrontEnd =
{
MaxFrontEndConcurrency = 1,
LogStatistics = false,
},
Schedule =
{
MaxIO = 1,
MaxProcesses = 1,
},
Sandbox =
{
FileSystemMode = FileSystemMode.RealAndMinimalPipGraph,
OutputReportingMode = OutputReportingMode.FullOutputOnError,
},
Logging =
{
LogsDirectory = logsPath,
LogStats = false,
LogExecution = false,
LogCounters = false,
LogMemory = false,
StoreFingerprints = false,
NoWarnings =
{
909, // Disable warnings about experimental feature
},
}
};
if (TryGetSubstSourceAndTarget(out string substSource, out string substTarget))
{
// Directory translation is needed here particularly when the test temporary directory
// is inside a directory that is actually a junction to another place.
// For example, the temporary directory is D:\src\BuildXL\Out\Object\abc\t_1, but
// the path D:\src\BuildXL\Out or D:\src\BuildXL\Out\Object is a junction to K:\Out.
// Some tool, like cmd, can access the path in K:\Out, and thus the test will have a DFA
// if there's no directory translation.
// This problem does not occur when only substs are involved, but no junctions. The method
// TryGetSubstSourceAndTarget works to get translations due to substs or junctions.
AbsolutePath substSourcePath = AbsolutePath.Create(Context.PathTable, substSource);
AbsolutePath substTargetPath = AbsolutePath.Create(Context.PathTable, substTarget);
Configuration.Engine.DirectoriesToTranslate.Add(
new TranslateDirectoryData(I($"{substSource}<{substTarget}"), substSourcePath, substTargetPath));
}
}
private (string host, ContentHash contentHash) ExtractHostHashFromAbsolutePath(AbsolutePath sourcePath)
{
// TODO: Keep the segments in the AbsolutePath object?
// TODO: Indexable structure?
var segments = sourcePath.GetSegments();
Contract.Assert(segments.Count >= 4);
string host = GetHostName(sourcePath.IsLocal, segments);
var hashLiteral = segments.Last();
if (hashLiteral.EndsWith(GrpcDistributedPathTransformer.BlobFileExtension, StringComparison.OrdinalIgnoreCase))
{
hashLiteral = hashLiteral.Substring(0, hashLiteral.Length - GrpcDistributedPathTransformer.BlobFileExtension.Length);
}
var hashTypeLiteral = segments.ElementAt(segments.Count - 1 - 2);
if (!Enum.TryParse(hashTypeLiteral, ignoreCase: true, out HashType hashType))
{
throw new InvalidOperationException($"{hashTypeLiteral} is not a valid member of {nameof(HashType)}");
}
var contentHash = new ContentHash(hashType, HexUtilities.HexToBytes(hashLiteral));
return(host, contentHash);
}
public override void PlaceFileStop(Context context, ContentHash contentHash, PlaceFileResult result, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, Severity successSeverity)
{
if (_eventSource.IsEnabled())
{
_eventSource.PlaceFileStop(context.Id.ToString(), (int)result.Code, result.ErrorMessage);
}
base.PlaceFileStop(context, contentHash, result, path, accessMode, replacementMode, realizationMode, successSeverity: DiagnosticLevelSeverity(result.Duration));
}
/// <summary>
/// Generate a basic file access manifest for front end tools
/// </summary>
public static FileAccessManifest GenerateToolFileAccessManifest(FrontEndContext context, AbsolutePath toolDirectory)
{
var pathTable = context.PathTable;
// We make no attempt at understanding what the tool is going to do
// We just configure the manifest to not fail on unexpected accesses, so they can be collected
// later if needed
var fileAccessManifest = new FileAccessManifest(pathTable)
{
FailUnexpectedFileAccesses = false,
ReportFileAccesses = true,
MonitorNtCreateFile = true,
MonitorZwCreateOpenQueryFile = true,
MonitorChildProcesses = true,
};
fileAccessManifest.AddScope(
AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.Windows)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(
AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.InternetCache)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(
AbsolutePath.Create(pathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.History)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(toolDirectory, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowReadAlways);
return(fileAccessManifest);
}
public void TrackerLoadFailureShouldNotResultInUnderbuild(bool updateFileContentTableByScanningChangeJournal)
{
// This test is created due to underbuild resulted from updating file content table during change journal scan.
// Here is the timeline of the scenario regarding source file 'srcA':
//
// Time FileContentTable FileChangeTracker FileSystem Description
// ----------------------------------------------------------------------------------------------------------------------------------
// T0 srcA:#1,USN1 Initial.
// T1 srcA:#1,USN1 Build P where srcA is read.
// No journal scan because no FileChangeTracker.
// T2 FID(srcA)->#1,USN1 FID(srcA)->USN1 srcA:#1,USN1 After building P: cache miss.
// T3 FID(srcA)->#1,USN1 FID(srcA)->USN1 srcA:#2,USN2 After srcA is modified on FileSystem.
// T4 FID(srcA)->#1,USN1 srcA:#2,USN2 FileChangeTracker is destroyed.
// T5 FID(srcA)->#1,USN1 srcA:#2,USN2 Build Q where srcA is only probed.
// No journal scan because no FileChangeTracker.
// T6 FID(srcA)->#1,USN1 FID(srcA)->USN2 srcA:#2,USN2 After building Q: cache miss.
// T7 FID(srcA)->#1,USN1 FID(srcA)->USN2 srcA:#2,USN3 Modified timestamp of srcA.
// T8 FID(srcA)->#1,USN1 FID(srcA)->USN2 srcA:#2,USN3 Build P again where srcA is read.
// T9 FID(srcA)->#1,USN3 FID(srcA)->USN2 srcA:#2,USN3 After journal scan, but before executing P.
// T10 FID(srcA)->#1,USN3 FID(srcA)->USN3 srcA:#2,USN3 After building P: cache hit.
//
// At T6, FileChangeTracker tracks srcA because srcA is probed.
// At T9 FileContentTable is updated based on the change journal scan from the checkpoint set at T6. From T6 to T8, there
// is only basic info change, i.e., timestamp change, that doesn't change the content hash of srcA. So, FID(srcA)'s entry
// in FileContentTable is eligible for update. However, if we simply update it with USN3, i.e., the current USN, then there's
// a mismatch between the content hash in FileContentTable and FileSystem that results in cache hit, i.e., underbuild.
//
// To handle the above case, we only update FileContentTable if the USN matches the one tracked by FileChangeTracker. In this case
// since FileChangeTracker has USN2 for srcA, and FileContentTable has USN1, then we should not update FileContentTable.
Configuration.Schedule.UpdateFileContentTableByScanningChangeJournal = updateFileContentTableByScanningChangeJournal;
AbsolutePath path = CreateUniqueSourcePath().Combine(Context.PathTable, "srcA");
FileArtifact srcA = CreateSourceFile(path);
// Populate file content table with srcA's content via P.
var pOperations = new[] { Operation.ReadFile(srcA), Operation.WriteFile(CreateOutputFileArtifact()) };
var p = CreateAndSchedulePipBuilder(pOperations);
var result = RunScheduler().AssertCacheMiss(p.Process.PipId);
// Modify srcA.
File.AppendAllText(ArtifactToString(srcA), Guid.NewGuid().ToString());
// Destroy file change tracker.
FileUtilities.DeleteFile(result.Config.Layout.SchedulerFileChangeTrackerFile.ToString(Context.PathTable), waitUntilDeletionFinished: true);
ResetPipGraphBuilder();
// Make srcA tracked and probe via Q.
var ssd = PipConstructionHelper.SealDirectorySource(path);
var qOperations = new[] { Operation.Probe(srcA, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) };
var qBuilder = CreatePipBuilder(qOperations);
qBuilder.AddInputDirectory(ssd);
var q = SchedulePipBuilder(qBuilder);
RunScheduler().AssertCacheMiss(q.Process.PipId);
// Modify basic file info.
var creationTime = File.GetCreationTime(ArtifactToString(srcA));
File.SetCreationTime(ArtifactToString(srcA), creationTime.Add(TimeSpan.FromSeconds(10)));
// Switch back to P.
ResetPipGraphBuilder();
p = CreateAndSchedulePipBuilder(pOperations);
// P should be miss because the content of srcA changed.
RunScheduler().AssertCacheMiss(p.Process.PipId);
}
/// <summary>
/// The FrontEnds that use an out-of-proc tool should sandbox that process and call this method
/// in order to tack the tool's file accesses, enumerations, etc. in order to make graph caching sound
/// </summary>
public static void TrackToolFileAccesses(FrontEndEngineAbstraction engine, FrontEndContext context, string frontEndName, ISet <ReportedFileAccess> fileAccesses, AbsolutePath frontEndFolder)
{
// Compute all parseable paths
// TODO: does it make sense to consider enumerations, or as a result the graph will be too unstable? Does it matter for MsBuild graph construction?
foreach (var access in fileAccesses)
{
string accessPath = access.GetPath(context.PathTable);
if (AbsolutePath.TryCreate(context.PathTable, accessPath, out AbsolutePath path))
{
// Ignore accesses under the frontend folder: these are files used for internal communication between
// BuildXL and the graph builder tool, and they are never files that MSBuild itself interacted with
if (path.IsWithin(context.PathTable, frontEndFolder))
{
continue;
}
if ((access.RequestedAccess & RequestedAccess.Enumerate) != 0)
{
engine.TrackDirectory(path.ToString(context.PathTable));
}
if ((access.RequestedAccess & RequestedAccess.Probe) != 0)
{
engine.FileExists(path);
}
if ((access.RequestedAccess & RequestedAccess.Read) != 0)
{
// Two things are happening here: we want to register if the file is present or absent. Engine.FileExists takes
// care of that. And in the case the file exists, record the content.
// There are apparently some repos that create and delete files during graph construction :(
// So we cannot trust detours and check for IsNonexistent on the access itself. Even though there were read/write accesses on a given file,
// the file may not exist at this point
if (engine.FileExists(path))
{
engine.RecordFrontEndFile(path, frontEndName);
}
}
}
}
}
protected override async Task <BoolResult> TryGetFileCoreAsync(OperationContext context, string storageId, AbsolutePath targetFilePath)
{
// Get the content from peers or fallback
var contentHashWithSize = await TryGetAndPutFileAsync(context, storageId, targetFilePath).ThrowIfFailureAsync();
// Register that the machine now has the content
await RegisterContent(context, contentHashWithSize);
return(BoolResult.Success);
}
private ExpandedAbsolutePath ChangeRoot(AbsolutePath path, PathTable pathTable, char newRoot)
{
XAssert.IsFalse(OperatingSystemHelper.IsUnixOS, "Cannot change root on a unix system");
return(AbsolutePath.Create(pathTable, newRoot + path.ToString(pathTable).Substring(1)).Expand(pathTable));
}
private async Task <Result <ContentHashWithSize> > TryGetAndPutFileAsync(OperationContext context, string storageId, AbsolutePath targetFilePath)
{
var(hash, fallbackStorageId) = ParseCompositeStorageId(storageId);
if (hash != null)
{
// First attempt to place file from content store
var placeResult = await _privateCas.PlaceFileAsync(context, hash.Value, targetFilePath, FileAccessMode.Write, FileReplacementMode.ReplaceExisting, FileRealizationMode.Copy, pinRequest : null);
if (placeResult.IsPlaced())
{
return(Result.Success(new ContentHashWithSize(hash.Value, placeResult.FileSize)));
}
// If not placed, try to copy from a peer into private CAS
var putResult = await CopyLocalAndPutAsync(context, hash.Value);
if (putResult.Succeeded)
{
// Lastly, try to place again now that file is copied to CAS
placeResult = await _privateCas.PlaceFileAsync(context, hash.Value, targetFilePath, FileAccessMode.Write, FileReplacementMode.ReplaceExisting, FileRealizationMode.Copy, pinRequest : null).ThrowIfFailure();
Counters[CentralStorageCounters.TryGetFileFromPeerSucceeded].Increment();
return(Result.Success(new ContentHashWithSize(hash.Value, placeResult.FileSize)));
}
else
{
Tracer.OperationDebug(context, $"Falling back to blob storage. Error={putResult}");
}
}
Counters[CentralStorageCounters.TryGetFileFromFallback].Increment();
return(await TryGetFromFallbackAndPutAsync(context, targetFilePath, fallbackStorageId));
}
public void ValidateCachingDirectoryEnumerationReadOnlyMountUntrackedScope(bool maskUntrackedAccesses)
{
// When true (default), directory enumerations in untracked scopes are ignored
// When false, directory enumerations in untracked scopes are tracked as normal
Configuration.Sandbox.MaskUntrackedAccesses = maskUntrackedAccesses;
AbsolutePath readonlyRootPath;
AbsolutePath.TryCreate(Context.PathTable, ReadonlyRoot, out readonlyRootPath);
DirectoryArtifact dir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(ReadonlyRoot));
// Enumerate untracked scope /dir and its parent directory
FileArtifact outFile = CreateOutputFileArtifact();
var ops = new Operation[]
{
Operation.EnumerateDir(DirectoryArtifact.CreateWithZeroPartialSealId(readonlyRootPath)),
Operation.EnumerateDir(dir),
Operation.WriteFile(outFile)
};
var builder = CreatePipBuilder(ops);
builder.AddUntrackedDirectoryScope(dir);
Process pip = SchedulePipBuilder(builder).Process;
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
string checkpoint1 = File.ReadAllText(ArtifactToString(outFile));
// Create /dir/nestedFile in untracked scope
FileArtifact nestedFile = CreateSourceFile(ArtifactToString(dir));
if (!maskUntrackedAccesses)
{
RunScheduler().AssertCacheMiss(pip.PipId);
}
RunScheduler().AssertCacheHit(pip.PipId);
// Modify /dir/nestedFile in untracked scope
File.WriteAllText(ArtifactToString(nestedFile), "nestedFile");
RunScheduler().AssertCacheHit(pip.PipId);
// Delete /dir/nestedFile in untracked scope
File.Delete(ArtifactToString(nestedFile));
RunScheduler().AssertCacheHit(pip.PipId);
string checkpoint2 = File.ReadAllText(ArtifactToString(outFile));
// Filesystem should match state from original run, so cache replays output from that run
XAssert.AreEqual(checkpoint1, checkpoint2);
// Create /dir/nestedDir in untracked scope
DirectoryArtifact nestedDir = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(ArtifactToString(dir)));
if (!maskUntrackedAccesses)
{
RunScheduler().AssertCacheMiss(pip.PipId);
}
RunScheduler().AssertCacheHit(pip.PipId);
// Delete /dir/nestedDir in untracked scope
Directory.Delete(ArtifactToString(nestedDir));
RunScheduler().AssertCacheHit(pip.PipId);
string checkpoint3 = File.ReadAllText(ArtifactToString(outFile));
// Filesystem should match state from original run, so cache replays output from that run
XAssert.AreEqual(checkpoint1, checkpoint3);
// Delete untracked scope /dir
Directory.Delete(ArtifactToString(dir));
RunScheduler().AssertCacheMiss(pip.PipId);
RunScheduler().AssertCacheHit(pip.PipId);
}
private async Task <ContentHash> PutAndRegisterFileAsync(OperationContext context, AbsolutePath file, ContentHash?hash)
{
PutResult putResult;
if (hash != null)
{
putResult = await _privateCas.PutFileAsync(context, file, FileRealizationMode.Copy, hash.Value, pinRequest : null).ThrowIfFailure();
}
else
{
putResult = await _privateCas.PutFileAsync(context, file, FileRealizationMode.Copy, _hashType, pinRequest : null).ThrowIfFailure();
}
var contentInfo = new ContentHashWithSize(putResult.ContentHash, putResult.ContentSize);
await RegisterContent(context, contentInfo);
return(putResult.ContentHash);
}
public override int Analyze()
{
HashSet <(CaseInsensitiveString toolPath, AbsolutePath directoryPath)> toolAndEnumerations = new HashSet <(CaseInsensitiveString toolPath, AbsolutePath directoryPath)>();
HashSet <AbsolutePath> enumeratedDirectories = new HashSet <AbsolutePath>();
HashSet <string> enumerationTools = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var observedInputEntry in m_observedInputsMap)
{
var pip = observedInputEntry.Key;
var observedInputs = observedInputEntry.Value;
if (!m_toolEnumerationMap.ContainsKey(pip))
{
continue;
}
var reportedFileAccesses = m_toolEnumerationMap[pip];
foreach (var observedInput in observedInputs)
{
if (observedInput.Type == ObservedInputType.DirectoryEnumeration)
{
enumeratedDirectories.Add(observedInput.Path);
}
}
foreach (var reportedFileAccess in reportedFileAccesses)
{
var path = reportedFileAccess.ManifestPath;
if (reportedFileAccess.Path != null)
{
if (!AbsolutePath.TryCreate(PathTable, reportedFileAccess.Path, out path))
{
continue;
}
}
if (!enumeratedDirectories.Contains(path))
{
continue;
}
enumerationTools.Add(reportedFileAccess.Process.Path);
toolAndEnumerations.Add((new CaseInsensitiveString(reportedFileAccess.Process.Path), path));
}
}
var toolEnumerationList = toolAndEnumerations.ToList();
toolEnumerationList.Sort((v1, v2) =>
{
var result = StringComparer.OrdinalIgnoreCase.Compare(v1.toolPath.Value, v2.toolPath.Value);
if (result != 0)
{
return(result);
}
return(PathTable.ExpandedPathComparer.Compare(v1.Item2, v2.Item2));
});
using (var outputStream = File.Create(OutputFilePath, bufferSize: 64 << 10 /* 64 KB */))
{
using (var writer = new StreamWriter(outputStream))
{
writer.WriteLine("Directory Enumeration Tools:");
foreach (var tool in enumerationTools.OrderBy(s => s, StringComparer.OrdinalIgnoreCase))
{
writer.WriteLine(tool);
}
writer.WriteLine();
writer.WriteLine("Enumerations By Tool:");
CaseInsensitiveString last = new CaseInsensitiveString(string.Empty);
foreach (var toolEnumerationEntry in toolEnumerationList)
{
var toolPath = toolEnumerationEntry.Item1;
var directoryPath = toolEnumerationEntry.Item2;
if (!last.Equals(toolPath))
{
last = toolPath;
writer.WriteLine();
writer.WriteLine("Tool: {0}", toolPath.Value);
}
writer.WriteLine(" Directory: {0}", directoryPath.ToString(PathTable));
}
toolEnumerationList.Sort((v1, v2) =>
{
var result = PathTable.ExpandedPathComparer.Compare(v1.Item2, v2.Item2);
if (result != 0)
{
return(result);
}
return(StringComparer.OrdinalIgnoreCase.Compare(v1.Item1.Value, v2.Item1.Value));
});
writer.WriteLine();
writer.WriteLine("Enumerations By Directory:");
AbsolutePath lastDirectory = AbsolutePath.Invalid;
foreach (var toolEnumerationEntry in toolEnumerationList)
{
var toolPath = toolEnumerationEntry.toolPath;
var directoryPath = toolEnumerationEntry.directoryPath;
if (!lastDirectory.Equals(directoryPath))
{
lastDirectory = directoryPath;
writer.WriteLine();
writer.WriteLine("Directory: {0}", directoryPath.ToString(PathTable));
}
writer.WriteLine(" Tool: {0}", toolPath.Value);
}
}
}
return(0);
}
/// <summary> /// <see cref="TouchBlobAsync"/> /// </summary> protected abstract Task <BoolResult> TouchBlobCoreAsync(OperationContext context, AbsolutePath file, string storageId, bool isUploader, bool isImmutable);
protected AbsolutePath Combine(AbsolutePath parent, string name)
{
return(parent.Combine(Context.PathTable, PathAtom.Create(Context.StringTable, name)));
}
/// <summary>
/// Try to get the blob specified by a given <paramref name="storageId"/> and store it in a <paramref name="targetFilePath"/>.
/// </summary>
public Task <BoolResult> TryGetFileAsync(OperationContext context, string storageId, AbsolutePath targetFilePath, bool isImmutable = false)
{
storageId = PreprocessStorageId(storageId);
return(context.PerformOperationAsync(
Tracer,
() => TryGetFileCoreAsync(context, storageId, targetFilePath, isImmutable),
counter: Counters[CentralStorageCounters.TryGetFile],
extraEndMessage: _ => $"StorageId=[{storageId}] TargetFilePath=[{targetFilePath}]",
traceOperationStarted: false));
}
public string GetAbsoluteImport()
{
return(Regex.Replace(AbsolutePath.Replace(_root, "").TrimStart('\\', '/').Replace("\\", "/"), @"\.ts$", ""));
}
