private async Task <bool> SendAttachCompletedAfterProcessBuildRequestStartedAsync() { if (!m_isGrpcEnabled) { #if !DISABLE_FEATURE_BOND_RPC m_bondMasterClient.Start(m_services, OnConnectionTimeOutAsync); #endif } var cacheValidationContent = Guid.NewGuid().ToByteArray(); var cacheValidationContentHash = ContentHashingUtilities.HashBytes(cacheValidationContent); var possiblyStored = await m_environment.Cache.ArtifactContentCache.TryStoreAsync( new MemoryStream(cacheValidationContent), cacheValidationContentHash); if (!possiblyStored.Succeeded) { Logger.Log.DistributionFailedToStoreValidationContentToWorkerCacheWithException( m_appLoggingContext, cacheValidationContentHash.ToHex(), possiblyStored.Failure.DescribeIncludingInnerFailures()); Exit(timedOut: true, "Failed to validate retrieve content from master via cache"); return(false); } var attachCompletionInfo = new AttachCompletionInfo { WorkerId = WorkerId, MaxConcurrency = m_maxProcesses, AvailableRamMb = m_scheduler.LocalWorker.TotalMemoryMb, WorkerCacheValidationContentHash = cacheValidationContentHash.ToBondContentHash(), }; Contract.Assert(attachCompletionInfo.WorkerCacheValidationContentHash != null, "worker cache validation content hash is null"); var attachCompletionResult = await m_masterClient.AttachCompletedAsync(attachCompletionInfo); if (!attachCompletionResult.Succeeded) { Logger.Log.DistributionInactiveMaster(m_appLoggingContext, (int)attachCompletionResult.Duration.TotalMinutes); Exit(timedOut: true, "Failed to attach to master"); return(true); } else { m_notifyMasterExecutionLogTarget = new NotifyMasterExecutionLogTarget(WorkerId, m_masterClient, m_environment.Context, m_scheduler.PipGraph.GraphId, m_scheduler.PipGraph.MaxAbsolutePathIndex, m_services); m_scheduler.AddExecutionLogTarget(m_notifyMasterExecutionLogTarget); m_sendThread.Start(); } return(true); }
private (FileArtifact, FileMaterializationInfo, PipOutputOrigin) CreateRandomOutputContent() { Random r = new Random(); var outputFile = CreateOutputFile(); var contentHash = ContentHashingUtilities.CreateRandom(); var fileContentInfo = new FileMaterializationInfo( new FileContentInfo(contentHash, r.Next(0, 102400)), outputFile.Path.GetName(Context.PathTable)); return(outputFile, fileContentInfo, PipOutputOrigin.Produced); }
private static Possible <ContentHash> TryHashFile(string path) { try { return(ContentHashingUtilities.HashFileAsync(path).Result); } catch (System.Exception ex) { return(new Failure <string>(I($"Failed to hash '{path}': {ex.GetLogEventMessage()}"))); } }
public void ValidateLengthSerializationRoundtrip() { var hash = ContentHashingUtilities.CreateRandom(); int length = 100; FileContentInfo original = new FileContentInfo(hash, length); var deserialized = new FileContentInfo(hash, FileContentInfo.LengthAndExistence.Deserialize(original.SerializedLengthAndExistence)); XAssert.IsTrue(original == deserialized); }
public async Task TestHashContentStreamAsync() { StreamWithLength stream = new MemoryStream(Encoding.UTF8.GetBytes("SampleString")).AssertHasLength(); ContentHash shaHash = await ContentHashingUtilities.HashContentStreamAsync(stream, HashType.SHA256); ContentHash vsoHash = await ContentHashingUtilities.HashContentStreamAsync(stream, HashType.Vso0); XAssert.AreEqual(shaHash.HashType, HashType.SHA256); XAssert.AreEqual(vsoHash.HashType, HashType.Vso0); }
public void DirectoryFingerprintEquality() { var fp1 = new DirectoryFingerprint(ContentHashingUtilities.CreateRandom()); var fp2 = fp1; var fp3 = new DirectoryFingerprint(ContentHashingUtilities.CreateRandom()); XAssert.IsTrue(fp1 == fp2); XAssert.IsFalse(fp1 != fp2); XAssert.IsTrue(fp2 != fp3); XAssert.IsFalse(fp2 == fp3); }
public async Task <ContentHash> StoreGuid(Guid guid) { var bytes = guid.ToByteArray(); var hash = ContentHashingUtilities.HashBytes(bytes); using (var stream = new MemoryStream(bytes)) { Analysis.IgnoreResult(await TryStoreAsync(stream, hash)); } return(hash); }
public async Task DeserializationReturnsNullIfUnavailable() { var cache = new InMemoryArtifactContentCache(); ContentHash imaginaryContent = ContentHashingUtilities.HashBytes(Encoding.UTF8.GetBytes("Imagination")); var maybeDeserialized = await cache.TryLoadAndDeserializeContent <PipCacheDescriptorV2Metadata>(imaginaryContent); XAssert.IsTrue(maybeDeserialized.Succeeded); XAssert.IsNull(maybeDeserialized.Result, "Should be a miss (cache empty)"); }
private async Task <ContentHash> GetContentHashAsync(string path, HashType hashType) { using var fs = FileUtilities.CreateFileStream( path, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read, FileOptions.SequentialScan); ContentHashingUtilities.SetDefaultHashType(hashType); return(await ContentHashingUtilities.HashContentStreamAsync(fs, hashType)); }
public void CreateSpecialValueGivesCorrectHash() { var contentHash = ContentHashingUtilities.CreateSpecialValue(7); var hashBytes = contentHash.ToHashByteArray(); for (var i = 1; i < ContentHashingUtilities.HashInfo.ByteLength; i++) { XAssert.AreEqual(0, hashBytes[i]); } XAssert.AreEqual(7, hashBytes[0]); }
private Task <Possible <ContentHash, Failure> > TryStoreInternalAsync( ExpandedAbsolutePath path, FileRealizationMode fileRealizationModes, ContentHash?knownContentHash) { return(Task.Run <Possible <ContentHash, Failure> >( () => { lock (m_lock) { byte[] contentBytes = ExceptionUtilities.HandleRecoverableIOException( () => { return File.ReadAllBytes(path.ExpandedPath); }, ex => { throw new BuildXLException("Failed to store content (couldn't read new content from disk)", ex); }); ContentHash contentHash = ContentHashingUtilities.HashBytes(contentBytes); if (knownContentHash.HasValue && contentHash != knownContentHash.Value) { return new Failure <string>(I($"Stored content had an unexpected hash. (expected: {knownContentHash.Value}; actual: {contentHash})")); } CacheEntry entry; if (m_content.TryGetValue(contentHash, out entry)) { // We assume that stores of content already present somewhere still cause replication // to both the local and remote sites. See class remarks. entry.Sites |= CacheSites.LocalAndRemote; return contentHash; } else { try { if (m_pathRealizationModes != null) { m_pathRealizationModes[path.ExpandedPath] = fileRealizationModes; } // We assume that stored content is instantly and magically replicated to some remote place. // See class remarks. m_content[contentHash] = new CacheEntry(contentBytes, CacheSites.LocalAndRemote); return contentHash; } catch (BuildXLException ex) { return new RecoverableExceptionFailure(ex); } } } })); }
private FakeFile CreateFakeFile(string path, string content) { FakeFile file = default(FakeFile); file.Path = path; // Make the content have a reasonable size so we can test getting split into multiple buffers when the backing // file is read into memory file.Content = Encoding.UTF8.GetBytes(Padding + content); file.Hash = ContentHashingUtilities.HashBytes(file.Content); return(file); }
public void CreateSpecialValueIsSpecial() { XAssert.IsTrue(ContentHashingUtilities.ZeroHash.IsSpecialValue()); XAssert.IsTrue(ContentHashingUtilities.CreateSpecialValue(1).IsSpecialValue()); XAssert.IsTrue(ContentHashingUtilities.CreateSpecialValue(2).IsSpecialValue()); XAssert.IsTrue(ContentHashingUtilities.CreateSpecialValue(3).IsSpecialValue()); // Technically, this could fail but if that ever happens its very strange for a random value to // actually end up being a special value var randomHash = ContentHashingUtilities.CreateRandom(); var randomHashIsSpecialValue = randomHash.IsSpecialValue(); XAssert.IsFalse(randomHashIsSpecialValue, "Random hash is a special value: '{0}'", randomHash.ToHex()); }
/// <summary> /// Returns a hash for the list of the file names, after case-normalizing them. /// </summary> private static DirectoryFingerprint CalculateDirectoryFingerprint(IReadOnlyList <string> fileNames) { if (fileNames.Count == 0) { return(DirectoryFingerprint.Zero); } string orderedFileNames = string.Join(",", fileNames.OrderBy(a => a, StringComparer.OrdinalIgnoreCase)); byte[] nameBytes = Encoding.Unicode.GetBytes(orderedFileNames.ToUpperInvariant()); var hash = ContentHashingUtilities.CreateFrom(MurmurHash3.Create(nameBytes, 0)); return(new DirectoryFingerprint(hash)); }
/// <summary> /// Returns a hash for the list of the file names, after case-normalizing them. /// </summary> private static DirectoryFingerprint CalculateDirectoryFingerprint(IReadOnlyList <string> fileNames) { if (fileNames.Count == 0) { return(DirectoryFingerprint.Zero); } string orderedFileNames = string.Join(",", fileNames.OrderBy(a => a, OperatingSystemHelper.PathComparer)); byte[] nameBytes = Encoding.Unicode.GetBytes(orderedFileNames.ToCanonicalizedPath()); var hash = ContentHashingUtilities.CreateFrom(MurmurHash3.Create(nameBytes, 0)); return(new DirectoryFingerprint(hash)); }
public async Task GetAndRecordContentHashAsyncWithEmptyFileContentTable() { var fileContentTable = FileContentTable.CreateNew(); WriteFile(FileA, "Some string"); FileContentTableExtensions.VersionedFileIdentityAndContentInfoWithOrigin result = await fileContentTable.GetAndRecordContentHashAsync(GetFullPath(FileA)); XAssert.AreEqual(FileContentTableExtensions.ContentHashOrigin.NewlyHashed, result.Origin); XAssert.AreEqual( await ContentHashingUtilities.HashFileAsync(GetFullPath(FileA)), result.VersionedFileIdentityAndContentInfo.FileContentInfo.Hash); }
public async Task OpenStreamToExistentContent() { ContentHash availableHash = await AddContent("Very useful data"); await LoadContentAndExpectAvailable(ContentCache, availableHash); Possible <Stream> maybeStream = await ContentCache.TryOpenContentStreamAsync(availableHash); XAssert.IsTrue(maybeStream.Succeeded); using (Stream stream = maybeStream.Result) { XAssert.AreEqual(availableHash, await ContentHashingUtilities.HashContentStreamAsync(stream)); } }
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null) { Contract.Requires(context != null); Contract.Requires(configuration != null); Context = context; LoggingContext = CreateLoggingContextForTest(); Configuration = configuration; FileContentTable = FileContentTable.CreateNew(); ContentFingerprinter = new PipContentFingerprinter( context.PathTable, artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo, ExtraFingerprintSalts.Default(), pathExpander: PathExpander); PipTable = pipTable; PipFragmentRenderer = this.CreatePipFragmentRenderer(); IpcProvider = IpcFactory.GetProvider(); var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext); Cache = InMemoryCacheFactory.Create(); LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker); m_sandboxConnectionKext = SandboxConnection; m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>(); m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>(); m_producers = new ConcurrentDictionary <FileArtifact, Pip>(); m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable); var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore); TempCleaner = new TestMoveDeleteCleaner(tempDirectory); State = new PipExecutionState( configuration, cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander), unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration, preserveOutputsSalt: ContentHashingUtilities.CreateRandom(), fileAccessWhitelist: FileAccessWhitelist, directoryMembershipFingerprinter: this, pathExpander: PathExpander, executionLog: null, fileSystemView: fileSystemView, fileContentManager: new FileContentManager(this, new NullOperationTracker()), directoryMembershipFinterprinterRuleSet: null); m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>(); ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable); }
private async Task <bool> SendAttachCompletedAfterProcessBuildRequestStartedAsync() { var cacheValidationContent = Guid.NewGuid().ToByteArray(); var cacheValidationContentHash = ContentHashingUtilities.HashBytes(cacheValidationContent); var possiblyStored = await m_environment.Cache.ArtifactContentCache.TryStoreAsync( new MemoryStream(cacheValidationContent), cacheValidationContentHash); if (!possiblyStored.Succeeded) { Logger.Log.DistributionFailedToStoreValidationContentToWorkerCacheWithException( m_appLoggingContext, cacheValidationContentHash.ToHex(), possiblyStored.Failure.DescribeIncludingInnerFailures()); Exit("Failed to validate retrieve content from master via cache", isUnexpected: true); return(false); } var attachCompletionInfo = new AttachCompletionInfo { WorkerId = WorkerId, MaxProcesses = m_config.Schedule.MaxProcesses, MaxMaterialize = m_config.Schedule.MaxMaterialize, AvailableRamMb = m_scheduler.LocalWorker.TotalRamMb, AvailableCommitMb = m_scheduler.LocalWorker.TotalCommitMb, WorkerCacheValidationContentHash = cacheValidationContentHash.ToBondContentHash(), }; Contract.Assert(attachCompletionInfo.WorkerCacheValidationContentHash != null, "worker cache validation content hash is null"); var attachCompletionResult = await m_masterClient.AttachCompletedAsync(attachCompletionInfo); if (!attachCompletionResult.Succeeded) { Exit($"Failed to attach to master. Duration: {(int)attachCompletionResult.Duration.TotalMinutes}", isUnexpected: true); return(true); } else { m_notifyMasterExecutionLogTarget = new NotifyMasterExecutionLogTarget(WorkerId, m_masterClient, m_environment.Context, m_scheduler.PipGraph.GraphId, m_scheduler.PipGraph.MaxAbsolutePathIndex, m_services); m_scheduler.AddExecutionLogTarget(m_notifyMasterExecutionLogTarget); m_sendThread.Start(); } return(true); }
private async Task <ContentHash> GetContentHashAsync(AbsolutePath path, HashType hashType = HashType.Unknown) { m_frontEndHost.Engine.RecordFrontEndFile(path, Name); // We don't call GetFileContentHashAsync() to get the existing hash, since it register the file. // This has been done in RecordFrontEndFile with the default hasher, re-register it with the specified hasher will cause error. using ( var fs = FileUtilities.CreateFileStream( path.ToString(m_context.PathTable), FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read, FileOptions.SequentialScan)) { return(await ContentHashingUtilities.HashContentStreamAsync(fs, hashType)); } }
public void DoubleWritePolicyIsContentAware(DoubleWritePolicy doubleWritePolicy) { BuildXLContext context = BuildXLContext.CreateInstanceForTesting(); var graph = new QueryablePipDependencyGraph(context); var analyzer = new TestFileMonitoringViolationAnalyzer( LoggingContext, context, graph, // Set this to test the logic of base.HandleDependencyViolation(...) instead of the overriding fake doLogging: true, collectNonErrorViolations: true); // Create the path where the double write will occur, and a random file content that will be used for both producers AbsolutePath doubleWriteOutput = CreateAbsolutePath(context, JunkPath); ContentHash contentHash = ContentHashingUtilities.CreateRandom(); var fileContentInfo = new FileContentInfo(contentHash, contentHash.Length); var outputsContent = new (FileArtifact, FileMaterializationInfo, PipOutputOrigin)[]
public void FileContentInfoEquality() { ContentHash hash1 = ContentHashingUtilities.CreateRandom(); ContentHash hash2 = hash1; ContentHash hash3 = ContentHashingUtilities.CreateRandom(); StructTester.TestEquality( baseValue: new FileContentInfo(hash1, 100), equalValue: new FileContentInfo(hash2, 100), notEqualValues: new[] { new FileContentInfo(hash3, 100), new FileContentInfo(hash2, 200), }, eq: (left, right) => left == right, neq: (left, right) => left != right); }
/// <summary> /// Compute a hash for our test streams /// </summary> /// <param name="testStream">The test stream to hash</param> /// <returns>The Hash of the contents of the stream</returns> /// <remarks> /// Our test streams can be rewound so this works as a way to get the hash /// </remarks> public static Hash AsHash(this Stream testStream) { Contract.Requires(testStream != null); byte[] contents = new byte[testStream.Length]; // We can only be sure we can do this for our test streams testStream.Seek(0, SeekOrigin.Begin); int read = testStream.Read(contents, 0, contents.Length); testStream.Seek(0, SeekOrigin.Begin); var contentHash = ContentHashingUtilities.HashBytes(contents); return(new Hash(contentHash)); }
/// <summary> /// Gets content hash. /// </summary> protected byte[] GetContentHash(FileArtifact file) { Contract.Requires(file.IsValid); byte[] hash = null; string fullPath = m_paths.Expand(file.Path); if (File.Exists(fullPath)) { var stream = new MemoryStream(File.ReadAllBytes(fullPath)); var contentHash = ContentHashingUtilities.HashContentStream(stream); return(contentHash.ToHashByteArray()); } return(hash); }
public TestData(PathTable pathTable, string testFolder) { TestContent = "ABCDEFG"; var uniqueFolder = Path.Combine(testFolder, Guid.NewGuid().ToString()); var sourceFolder = Path.Combine(uniqueFolder, "server"); Directory.CreateDirectory(sourceFolder); SourceFile = Path.Combine(sourceFolder, "file.txt"); File.WriteAllText(SourceFile, TestContent); SourceHash = ContentHashingUtilities.HashFileAsync(SourceFile).Result; var targetFolder = Path.Combine(uniqueFolder, "test"); TargetFile = Path.Combine(targetFolder, "file.txt"); TargetPath = AbsolutePath.Create(pathTable, TargetFile); }
/// <summary> /// Serializes a path set to the given buffer. /// </summary> protected async Task <ContentHash> SerializePathSet(ObservedPathSet pathSet, MemoryStream pathSetBuffer, ContentHash?pathSetHash = null) { using (var writer = new BuildXLWriter(stream: pathSetBuffer, debug: false, leaveOpen: true, logStats: false)) { pathSet.Serialize(PathTable, writer, m_pathExpander); if (pathSetHash == null) { pathSetBuffer.Position = 0; pathSetHash = await ContentHashingUtilities.HashContentStreamAsync(pathSetBuffer); } pathSetBuffer.Position = 0; return(pathSetHash.Value); } }
public void ExistenceAndHasKnownLengthBehavior() { // non-empty hash + non-zero length => exists as a file var fci = new FileContentInfo(ContentHashingUtilities.CreateRandom(), 100); XAssert.IsTrue(fci.HasKnownLength); XAssert.IsTrue(fci.Existence.HasValue && fci.Existence.Value == PathExistence.ExistsAsFile); // empty hash + zero length => exists as a file fci = new FileContentInfo(ContentHashingUtilities.EmptyHash, 0); XAssert.IsTrue(fci.HasKnownLength); XAssert.IsTrue(fci.Existence.HasValue && fci.Existence.Value == PathExistence.ExistsAsFile); // empty hash + non-zero length => undefined fci = new FileContentInfo(ContentHashingUtilities.EmptyHash, 1); XAssert.IsFalse(fci.HasKnownLength); XAssert.IsFalse(fci.Existence.HasValue); // if the existence was not explicitly set, it should not magically appear fci = FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.CreateRandom()); XAssert.IsFalse(fci.HasKnownLength); XAssert.IsFalse(fci.Existence.HasValue); // we should see exactly the same value that was passed when the struct was created var existence = PathExistence.ExistsAsDirectory; fci = FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.CreateRandom(), existence); XAssert.IsFalse(fci.HasKnownLength); XAssert.IsTrue(fci.Existence.HasValue && fci.Existence.Value == existence); // if a special hash is used, the length is invalid var specialHash = ContentHashingUtilities.CreateSpecialValue(1); fci = new FileContentInfo(specialHash, 100); XAssert.IsFalse(fci.HasKnownLength); XAssert.IsFalse(fci.Existence.HasValue); fci = FileContentInfo.CreateWithUnknownLength(specialHash); XAssert.IsFalse(fci.HasKnownLength); XAssert.IsFalse(fci.Existence.HasValue); fci = FileContentInfo.CreateWithUnknownLength(specialHash, existence); XAssert.IsFalse(fci.HasKnownLength); XAssert.IsTrue(fci.Existence.HasValue && fci.Existence.Value == existence); }
private Tuple <Fingerprint, string> ComputeTimestampBasedHashInternal(bool skipManifestCheckTestHook) { Stopwatch sw = Stopwatch.StartNew(); // Computes a hash based on the paths and timestamps of all of the referenced files using (var wrapper = Pools.StringBuilderPool.GetInstance()) { StringBuilder sb = wrapper.Instance; foreach (var file in GetRelevantRelativePaths(forServerDeployment: true)) { try { FileInfo fi = new FileInfo(Path.Combine(BaseDirectory, file)); sb.Append(fi.Name); sb.Append(':'); sb.Append(fi.LastWriteTimeUtc.ToBinary()); sb.AppendLine(); } #pragma warning disable ERP022 // TODO: This should really handle specific errors catch { // noop for files that cannot be found. The manifest will include exteraneous files } #pragma warning restore ERP022 // Unobserved exception in generic exception handler } if (!skipManifestCheckTestHook) { ContentHashingUtilities.SetDefaultHashType(); AddHashForManifestFile(sb); } if (sb.Length == 0) { throw new BuildXLException("App Deployment hash could not be computed because no files from the deployment manifest could be accessed."); } string text = sb.ToString(); Fingerprint fingerprint = FingerprintUtilities.Hash(text); ComputeTimestampBasedHashTime = sw.Elapsed; return(new Tuple <Fingerprint, string>(fingerprint, text)); } }
public static ObservedPathSet CreatePathSet(PathTable pathTable, params AbsolutePath[] paths) { ObservedPathEntry[] entries = paths.Select(p => new ObservedPathEntry(p, false, false, false, null, false)).ToArray(); SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> sortedPathIds = SortedReadOnlyArray <ObservedPathEntry, ObservedPathEntryExpandedPathComparer> .SortUnsafe( entries, new ObservedPathEntryExpandedPathComparer(pathTable.ExpandedPathComparer)); var emptyObservedAccessFileNames = SortedReadOnlyArray <StringId, CaseInsensitiveStringIdComparer> .FromSortedArrayUnsafe( ReadOnlyArray <StringId> .Empty, new CaseInsensitiveStringIdComparer(pathTable.StringTable)); return(new ObservedPathSet( sortedPathIds, emptyObservedAccessFileNames, new UnsafeOptions(UnsafeOptions.SafeConfigurationValues, ContentHashingUtilities.CreateRandom()))); }
/// <summary> /// Computes the hash of all the specified search path tool suffixes /// </summary> public ContentHash?ComputeSearchPathToolsHash() { if (m_searchPathEnumerationToolFragments == null || m_searchPathEnumerationToolFragments.Count == 0) { return(null); } var hash = ContentHashingUtilities.ZeroHash; foreach (var searchPathToolSuffix in m_searchPathEnumerationToolFragments) { hash = ContentHashingUtilities.CombineOrderIndependent( hash, ContentHashingUtilities.HashString(searchPathToolSuffix.ToUpperInvariant())); } return(hash); }