public Task TestColdStorage() { return(RunTestAsync(async(context, store, directory) => { var originalPath = directory.Path / "original.txt"; var fileContents = GetRandomFileContents(); // Create the file and hardlink it into the cache. FileSystem.WriteAllText(originalPath, fileContents); var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5); var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents)); await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess(); // Hardlink back to original location trying to replace existing. var copyResult = await store.PlaceFileAsync( context, contentHash, originalPath, FileAccessMode.ReadOnly, FileReplacementMode.ReplaceExisting, FileRealizationMode.Any, CancellationToken.None).ShouldBeSuccess(); // The file is intact. FileSystem.ReadAllText(originalPath).Should().Be(fileContents); })); }
public void SerializeToStringReverse(HashType hashType, string expected) { var hashLength = HashInfoLookup.Find(hashType).ByteLength; var hash = new ContentHash(hashType, Enumerable.Range(0, hashLength).Select(i => (byte)i).ToArray()); Assert.Equal(expected, hash.SerializeReverse()); }
public async Task PushIsRejectedForTheSameHash() { await Task.Yield(); int numberOfPushes = 100; await RunTestCase(async (rootPath, session, client) => { var bytes = ThreadSafeRandom.GetBytes(1 + 42); var input = Enumerable.Range(1, numberOfPushes) .Select(data => (stream: new MemoryStream(bytes), hash: HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(bytes))) .ToList(); var pushTasks = input.Select( tpl => client.PushFileAsync( new OperationContext(_context), tpl.hash, tpl.stream, new CopyOptions(bandwidthConfiguration: null))).ToList(); var results = await Task.WhenAll(pushTasks); results.Any(r => r.Status == CopyResultCode.Rejected_OngoingCopy).Should().BeTrue(); var result = await client.PushFileAsync( new OperationContext(_context), input[0].hash, input[0].stream, new CopyOptions(bandwidthConfiguration: null)); result.Status.Should().Be(CopyResultCode.Rejected_ContentAvailableLocally); }); }
private static ContentHash ComputeEmptyVsoHash() { using (var hasher = HashInfoLookup.Find(ContentHashType).CreateContentHasher()) { return(hasher.GetContentHash(new byte[] { })); } }
/// <nodoc /> public DistributedContentCopier( AbsolutePath workingDirectory, DistributedContentStoreSettings settings, IAbsFileSystem fileSystem, IFileCopier <T> fileCopier, IFileExistenceChecker <T> fileExistenceChecker, ICopyRequester copyRequester, IPathTransformer <T> pathTransformer, IContentLocationStore contentLocationStore) { Contract.Requires(settings != null); Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1); _settings = settings; _tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp"); _remoteFileCopier = fileCopier; _remoteFileExistenceChecker = fileExistenceChecker; _copyRequester = copyRequester; _contentLocationStore = contentLocationStore; _pathTransformer = pathTransformer; _fileSystem = fileSystem; _workingDirectory = _tempFolderForCopies.Path; // TODO: Use hashers from IContentStoreInternal instead? _hashers = HashInfoLookup.CreateAll(); _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations); _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations); _retryIntervals = settings.RetryIntervalForCopies; _timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies; }
public Task SimpleBlobPutAndGetTest() { return(RunTest(async(context, service, iteration) => { // First heartbeat lets the service know its master, so it's willing to process requests await service.OnRoleUpdatedAsync(context, Role.Master); var machineId = new MachineId(0); var data = ThreadSafeRandom.GetBytes((int)100); var contentHash = HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(data); var putResponse = await service.PutBlobAsync(new PutBlobRequest() { ContextId = Guid.NewGuid().ToString(), ContentHash = contentHash, Blob = data }); putResponse.Succeeded.Should().BeTrue(); var getResponse = await service.GetBlobAsync(new GetBlobRequest() { ContextId = Guid.NewGuid().ToString(), ContentHash = contentHash, }); getResponse.Succeeded.Should().BeTrue(); getResponse.Blob.Should().NotBeNull(); getResponse.Blob.Should().BeEquivalentTo(data); })); }
public void CreateContentHasher() { var hashInfo = HashInfoLookup.Find(_hasher.Info.HashType); // Warm-up session. PerfTestCase(); var stopwatch = Stopwatch.StartNew(); for (var i = 0; i < Iterations; i++) { PerfTestCase(); } stopwatch.Stop(); var rate = (long)(Iterations / stopwatch.Elapsed.TotalSeconds); var name = GetType().Name + "." + nameof(CreateContentHasher); _resultsFixture.AddResults(Output, name, rate, "items/sec", Iterations); void PerfTestCase() { using (var hasher = hashInfo.CreateContentHasher()) { hasher.Info.HashType.Should().Be(_hasher.Info.HashType); } } }
public static async Task <PutResult> PutRandomFileAsync( this IContentSession session, Context context, IAbsFileSystem fileSystem, HashType hashType, bool provideHash, long size, CancellationToken ct) { Contract.RequiresNotNull(session); Contract.RequiresNotNull(context); Contract.RequiresNotNull(fileSystem); using (var directory = new DisposableDirectory(fileSystem)) { var c = context.CreateNested(); // TODO: Fix this to work with size > int.Max (bug 1365340) var data = ThreadSafeRandom.GetBytes((int)size); var path = directory.CreateRandomFileName(); fileSystem.WriteAllBytes(path, data); if (!provideHash) { return(await session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct).ConfigureAwait(false)); } var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(await session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct).ConfigureAwait(false)); } }
/// <summary> /// Put a randomly-sized content from a file into the store. /// </summary> public static Task <PutResult> PutRandomFileAsync( this IContentSession session, Context context, IAbsFileSystem fileSystem, AbsolutePath path, HashType hashType, bool provideHash, long size, CancellationToken ct) { Contract.RequiresNotNull(session); Contract.RequiresNotNull(context); Contract.RequiresNotNull(fileSystem); var c = context.CreateNested(nameof(ContentSessionExtensions)); // TODO: Fix this to work with size > int.Max (bug 1365340) var data = ThreadSafeRandom.GetBytes((int)size); fileSystem.WriteAllBytes(path, data); if (!provideHash) { return(session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct)); } var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct)); }
public static async Task <PutResult> PutRandomAsync( this IContentSession session, Context context, HashType hashType, bool provideHash, long size, CancellationToken ct) { Contract.RequiresNotNull(session); Contract.RequiresNotNull(context); var c = context.CreateNested(); // TODO: Fix this to work with size > int.Max (bug 1365340) var data = ThreadSafeRandom.GetBytes((int)size); using (var stream = new MemoryStream(data)) { if (!provideHash) { return(await session.PutStreamAsync(c, hashType, stream, ct).ConfigureAwait(false)); } var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(await session.PutStreamAsync(c, hash, stream, ct).ConfigureAwait(false)); } }
public Task HandlesMultipleHashTypes() { var context = new Context(Logger); return(TestContentDirectory(context, async contentDirectory => { using (var sha1Hasher = HashInfoLookup.Find(HashType.SHA1).CreateContentHasher()) { using (var sha256Hasher = HashInfoLookup.Find(HashType.SHA256).CreateContentHasher()) { byte[] content = ThreadSafeRandom.GetBytes(DefaultFileSize); ContentHash sha1ContentHash = sha1Hasher.GetContentHash(content); ContentHash sha256ContentHash = sha256Hasher.GetContentHash(content); await contentDirectory.UpdateAsync(sha1ContentHash, true, MemoryClock, info => { Assert.Null(info); return Task.FromResult(new ContentFileInfo(MemoryClock, 1, content.Length)); }); await contentDirectory.UpdateAsync(sha256ContentHash, true, MemoryClock, info => { Assert.Null(info); return Task.FromResult(new ContentFileInfo(MemoryClock, 2, content.Length)); }); var hashes = (await contentDirectory.EnumerateContentHashesAsync()).ToList(); hashes.Contains(sha1ContentHash).Should().BeTrue(); hashes.Contains(sha256ContentHash).Should().BeTrue(); } } })); }
public ColdStorage(IAbsFileSystem fileSystem, ColdStorageSettings coldStorageSettings, DistributedContentCopier distributedContentCopier) { _fileSystem = fileSystem; _copier = distributedContentCopier; _rootPath = coldStorageSettings.GetAbsoulutePath(); ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(coldStorageSettings.CacheSizeQuotaString !))); ContentStoreSettings contentStoreSettings = FromColdStorageSettings(coldStorageSettings); _store = new FileSystemContentStore(fileSystem, SystemClock.Instance, _rootPath, configurationModel, null, contentStoreSettings, null); HashType hashType; if (!Enum.TryParse <HashType>(coldStorageSettings.ConsistentHashingHashType, true, out hashType)) { hashType = HashType.SHA256; } _contentHasher = HashInfoLookup.GetContentHasher(hashType); _copiesQuantity = coldStorageSettings.ConsistentHashingCopiesQuantity; _maxParallelPlaces = coldStorageSettings.MaxParallelPlaces; // Starts empty and is created during the first update _ring = new RingNode[0]; }
public void ToHexCorrect(HashType hashType, string expected) { var hashLength = HashInfoLookup.Find(hashType).ByteLength; var hash = new ContentHash(hashType, Enumerable.Range(0, hashLength).Select(i => (byte)i).ToArray()); Assert.Equal(expected, hash.ToHex()); }
public async Task HashBytesOfVariousSizes(int size) { using var fileSystem = new PassThroughFileSystem(); using var tempDirectory = new DisposableDirectory(fileSystem); var content = ThreadSafeRandom.GetBytes(size); var path = tempDirectory.Path / "1.txt"; fileSystem.WriteAllBytes(path, content); var hashType = HashType.Vso0; var contentHasher = HashInfoLookup.GetContentHasher(hashType); var h1 = contentHasher.GetContentHash(content); var h2 = CalculateHashWithMemoryMappedFile(fileSystem, path, hashType); Assert.Equal(h1, h2); #if NET_COREAPP h2 = contentHasher.GetContentHash(content.AsSpan()); Assert.Equal(h1, h2); #endif using var memoryStream = new MemoryStream(content); var h3 = await contentHasher.GetContentHashAsync(memoryStream); Assert.Equal(h1, h3); // Using an old style hashing to make sure it works the same as the new and optimized version. using var fileStream = fileSystem.OpenForHashing(path); var h4 = await contentHasher.GetContentHashAsync(fileStream); Assert.Equal(h1, h4); }
public void VsoContentHasherAlgorithmUsesVsoHashInfo() { using (var hasher = HashInfoLookup.Find(HashType.Vso0).CreateContentHasher()) { Assert.Equal(VsoHashInfo.Instance.ByteLength, hasher.Info.ByteLength); Assert.Equal(VsoHashInfo.Instance.HashType.ToString(), hasher.Info.Name); Assert.Equal(VsoHashInfo.Instance.HashType, hasher.Info.HashType); } }
private static IContentHasher GetContentHasher(HashType hashType) { if (hashType == HashInfo.HashType || hashType == HashType.Unknown) { return(s_hasher); } return(s_contentHasherByHashType.GetOrAdd(hashType, (_) => HashInfoLookup.Find(hashType).CreateContentHasher())); }
/// <nodoc /> internal static async Task <DedupNode> GetDedupNodeFromFileAsync(HashType hashType, string path) { var contentHasher = (DedupContentHasher <DedupNodeOrChunkHashAlgorithm>)HashInfoLookup.GetContentHasher(hashType); using (var stream = FileStreamUtility.OpenFileStreamForAsync(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { return(await contentHasher.HashContentAndGetDedupNodeAsync(stream)); } }
public void ToHashBytesArray(HashType hashType) { var length = HashInfoLookup.Find(hashType).ByteLength; var bytes = Enumerable.Range(0, length).Select(i => (byte)i).ToArray(); var contentHash = new ContentHash(hashType, bytes); var exported = contentHash.ToHashByteArray(); Assert.Equal(bytes, exported); }
/// <inheritdoc /> public async Task <PutResult> PutFileAsync( Context context, HashType hashType, AbsolutePath path, FileRealizationMode realizationMode, CancellationToken cts, UrgencyHint urgencyHint) { if (hashType != RequiredHashType) { return(new PutResult( new ContentHash(hashType), $"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{hashType}'.")); } try { long contentSize; ContentHash contentHash; using (var hashingStream = new FileStream( path.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, StreamBufferSize)) { contentSize = hashingStream.Length; contentHash = await HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHashAsync(hashingStream).ConfigureAwait(false); } using (var streamToPut = FileStreamUtils.OpenFileStreamForAsync( path.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { BoolResult putSucceeded = await PutLazyStreamAsync( context, contentHash, streamToPut, cts, urgencyHint).ConfigureAwait(false); if (!putSucceeded.Succeeded) { return(new PutResult( putSucceeded, contentHash, $"Failed to add a BlobStore reference to content with hash=[{contentHash}]")); } } return(new PutResult(contentHash, contentSize)); } catch (Exception e) { return(new PutResult(e, new ContentHash(hashType))); } }
private ContentHash ComputeStartedCopyHash(ContentHash hash) { var murmurHash = MurmurHash3.Create(hash.ToByteArray(), _startedCopyHashSeed); var hashLength = HashInfoLookup.Find(_hashType).ByteLength; var buffer = murmurHash.ToByteArray(); Array.Resize(ref buffer, hashLength); return new ContentHash(_hashType, buffer); }
internal void CopyFileTo( [Required, Description("Machine to copy to")] string host, [Required, Description("Path to source file")] string sourcePath, [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified")] string grpcPortFileName, [Description("The GRPC port"), DefaultValue(0)] int grpcPort) { Initialize(); var context = new Context(_logger); var operationContext = new OperationContext(context, CancellationToken.None); var retryPolicy = RetryPolicyFactory.GetLinearPolicy(ex => ex is ClientCanRetryException, (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds)); if (grpcPort == 0) { grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName); } var hasher = HashInfoLookup.GetContentHasher(HashType.MD5); var bytes = File.ReadAllBytes(sourcePath); var hash = hasher.GetContentHash(bytes); try { var path = new AbsolutePath(sourcePath); using Stream stream = File.OpenRead(path.Path); var config = new GrpcCopyClientConfiguration(); using var clientCache = new GrpcCopyClientCache(context, new GrpcCopyClientCacheConfiguration() { GrpcCopyClientConfiguration = config }); var copyFileResult = clientCache.UseAsync(operationContext, host, grpcPort, (nestedContext, rpcClient) => { return(retryPolicy.ExecuteAsync( () => rpcClient.PushFileAsync(nestedContext, hash, stream, new CopyOptions(bandwidthConfiguration: null)), _cancellationToken)); }).GetAwaiter().GetResult(); if (!copyFileResult.Succeeded) { _tracer.Error(context, $"{copyFileResult}"); throw new CacheException(copyFileResult.ErrorMessage); } else { _tracer.Info(context, $"Copy of {sourcePath} was successful"); } } catch (Exception ex) { throw new CacheException(ex.ToString()); } }
public static bool IsValidLength(long length, ContentHash hash) { if (length < 0) { return(false); } ContentHash emptyHash = HashInfoLookup.Find(hash.HashType).EmptyHash; return(length == 0 ? (hash == emptyHash) : (hash != emptyHash)); }
/// <inheritdoc /> protected override async Task <PutResult> PutStreamCoreAsync( OperationContext context, HashType hashType, Stream stream, UrgencyHint urgencyHint, Counter retryCounter) { if (hashType != RequiredHashType) { return(new PutResult( new ContentHash(hashType), $"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{hashType}'.")); } try { StreamWithLength streamToPut; // Can't assume we've been given a seekable stream. if (stream.CanSeek) { streamToPut = stream.AssertHasLength(); } else { streamToPut = await CreateSeekableStreamAsync(context, stream); } using (streamToPut) { Contract.Assert(streamToPut.Stream.CanSeek); long contentSize = streamToPut.Length; var contentHash = await HashInfoLookup.Find(hashType) .CreateContentHasher() .GetContentHashAsync(streamToPut) .ConfigureAwait(false); streamToPut.Stream.Seek(0, SeekOrigin.Begin); var putResult = await PutLazyStreamAsync(context, contentHash, streamToPut, urgencyHint).ConfigureAwait(false); if (!putResult.Succeeded) { return(new PutResult(putResult, contentHash, $"Failed to add a BlobStore reference to content with hash=[{contentHash}]")); } return(new PutResult(contentHash, contentSize)); } } catch (Exception e) { return(new PutResult(e, new ContentHash(hashType))); } }
public void Indexer(HashType hashType) { var length = HashInfoLookup.Find(hashType).ByteLength; var bytes = Enumerable.Range(0, length).Select(i => (byte)i).ToArray(); var contentHash = new ContentHash(hashType, bytes); foreach (var i in Enumerable.Range(0, length)) { var b = contentHash[i]; Assert.Equal((byte)i, b); } }
public static bool IsValidLength(long length, ContentHash hash) { if (length < 0 || length > LengthAndExistence.MaxSupportedLength || hash.IsSpecialValue()) { return(false); } ContentHash emptyHash = HashInfoLookup.Find(hash.HashType).EmptyHash; return(length == 0 ? (hash == emptyHash) : (hash != emptyHash)); }
public void EqualContentHashRoundTripViaHexString(HashType hashType) { var h1 = ContentHash.Random(hashType); var hex = h1.ToHex(); var hashInfo = HashInfoLookup.Find(hashType); var buffer = new byte[hashInfo.ByteLength]; var sb = HexUtilities.HexToBytes(hex, buffer); var h2 = new ContentHash(hashType, sb); Assert.Equal(h1, h2); }
public void RandomValue(HashType hashType) { var v = ContentHash.Random(hashType); Assert.Equal(hashType, v.HashType); var hashInfo = HashInfoLookup.Find(hashType); if (hashInfo is TaggedHashInfo taggedHashInfo) { Assert.Equal(v[hashInfo.ByteLength - 1], taggedHashInfo.AlgorithmId); } }
private async Task <bool> ValidateNameHashesMatchContentHashesAsync(Context context) { int mismatchedParentDirectoryCount = 0; int mismatchedContentHashCount = 0; _tracer.Always(context, "Validating local CAS content hashes..."); await TaskUtilities.SafeWhenAll(_enumerateBlobPathsFromDisk().Select( async blobPath => { var contentFile = blobPath.FullPath; if (!contentFile.FileName.StartsWith(contentFile.GetParent().FileName, StringComparison.OrdinalIgnoreCase)) { mismatchedParentDirectoryCount++; _tracer.Debug( context, $"The first {FileSystemContentStoreInternal.HashDirectoryNameLength} characters of the name of content file at {contentFile}" + $" do not match the name of its parent directory {contentFile.GetParent().FileName}."); } if (!FileSystemContentStoreInternal.TryGetHashFromPath(context, _tracer, contentFile, out var hashFromPath)) { _tracer.Debug( context, $"The path '{contentFile}' does not contain a well-known hash name."); return; } var hasher = HashInfoLookup.GetContentHasher(hashFromPath.HashType); ContentHash hashFromContents; using (var contentStream = _fileSystem.Open( contentFile, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashingExtensions.HashStreamBufferSize)) { hashFromContents = await hasher.GetContentHashAsync(contentStream); } if (hashFromContents != hashFromPath) { mismatchedContentHashCount++; _tracer.Debug( context, $"Content at {contentFile} content hash {hashFromContents.ToShortString()} did not match expected value of {hashFromPath.ToShortString()}."); } })); _tracer.Always(context, $"{mismatchedParentDirectoryCount} mismatches between content file name and parent directory."); _tracer.Always(context, $"{mismatchedContentHashCount} mismatches between content file name and file contents."); return(mismatchedContentHashCount == 0 && mismatchedParentDirectoryCount == 0); }
public static async Task <PutResult> PutContentAsync( this IContentSession session, Context context, string content) { var c = context.CreateNested(); var data = Encoding.UTF8.GetBytes(content); var hashType = HashType.SHA256; using (var stream = new MemoryStream(data)) { var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(await session.PutStreamAsync(c, hash, stream, CancellationToken.None).ConfigureAwait(false)); } }
public Task TestColdStorageWithBulkFunction() { return(RunTestAsync(async(context, store, directory) => { var originalPath = directory.Path / "original.txt"; var fileContents = GetRandomFileContents(); // Build destination IContentSession DisposableDirectory sessionDirectory = new DisposableDirectory(FileSystem); ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("10MB"))); FileSystemContentStore destination = new FileSystemContentStore(FileSystem, SystemClock.Instance, sessionDirectory.Path, configurationModel); _ = await destination.StartupAsync(context); IContentSession contentSession = destination.CreateSession(context, "test_session", BuildXL.Cache.ContentStore.Interfaces.Stores.ImplicitPin.None).Session; _ = await contentSession.StartupAsync(context); // Create the file and hardlink it into the cache. FileSystem.WriteAllText(originalPath, fileContents); var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5); var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents)); await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess(); FileSystem.DeleteFile(originalPath); FileSystem.FileExists(originalPath).Should().Be(false); ContentHashWithPath contentHashWithPath = new ContentHashWithPath(contentHash, originalPath); List <ContentHashWithPath> listFile = new List <ContentHashWithPath>(); listFile.Add(contentHashWithPath); // Hardlink back to original location trying to replace existing. var copyTask = await store.FetchThenPutBulkAsync( context, listFile, contentSession); await copyTask.ToLookupAwait(r => { return r.Item.Succeeded; }); FileSystem.FileExists(originalPath).Should().Be(false); // The file is in the destination. await contentSession.PlaceFileAsync(context, contentHash, originalPath, FileAccessMode.Write, FileReplacementMode.FailIfExists, FileRealizationMode.Copy, CancellationToken.None).ShouldBeSuccess(); FileSystem.FileExists(originalPath).Should().Be(true); FileSystem.ReadAllText(originalPath).Should().Be(fileContents); _ = await contentSession.ShutdownAsync(context); _ = await destination.ShutdownAsync(context); })); }