コード例 #1
0
        public Task TestColdStorage()
        {
            return(RunTestAsync(async(context, store, directory) =>
            {
                var originalPath = directory.Path / "original.txt";
                var fileContents = GetRandomFileContents();

                // Create the file and hardlink it into the cache.
                FileSystem.WriteAllText(originalPath, fileContents);

                var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
                var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));

                await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();

                // Hardlink back to original location trying to replace existing.
                var copyResult = await store.PlaceFileAsync(
                    context,
                    contentHash,
                    originalPath,
                    FileAccessMode.ReadOnly,
                    FileReplacementMode.ReplaceExisting,
                    FileRealizationMode.Any,
                    CancellationToken.None).ShouldBeSuccess();

                // The file is intact.
                FileSystem.ReadAllText(originalPath).Should().Be(fileContents);
            }));
        }
コード例 #2
0
        public void SerializeToStringReverse(HashType hashType, string expected)
        {
            var hashLength = HashInfoLookup.Find(hashType).ByteLength;
            var hash       = new ContentHash(hashType, Enumerable.Range(0, hashLength).Select(i => (byte)i).ToArray());

            Assert.Equal(expected, hash.SerializeReverse());
        }
コード例 #3
0
        public async Task PushIsRejectedForTheSameHash()
        {
            await Task.Yield();

            int numberOfPushes = 100;

            await RunTestCase(async (rootPath, session, client) =>
            {
                var bytes = ThreadSafeRandom.GetBytes(1 + 42);
                var input = Enumerable.Range(1, numberOfPushes)
                            .Select(data => (stream: new MemoryStream(bytes), hash: HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(bytes)))
                            .ToList();

                var pushTasks = input.Select(
                    tpl =>
                    client.PushFileAsync(
                        new OperationContext(_context),
                        tpl.hash,
                        tpl.stream,
                        new CopyOptions(bandwidthConfiguration: null))).ToList();

                var results = await Task.WhenAll(pushTasks);

                results.Any(r => r.Status == CopyResultCode.Rejected_OngoingCopy).Should().BeTrue();

                var result = await client.PushFileAsync(
                    new OperationContext(_context),
                    input[0].hash,
                    input[0].stream,
                    new CopyOptions(bandwidthConfiguration: null));
                result.Status.Should().Be(CopyResultCode.Rejected_ContentAvailableLocally);
            });
        }
 private static ContentHash ComputeEmptyVsoHash()
 {
     using (var hasher = HashInfoLookup.Find(ContentHashType).CreateContentHasher())
     {
         return(hasher.GetContentHash(new byte[] { }));
     }
 }
コード例 #5
0
        /// <nodoc />
        public DistributedContentCopier(
            AbsolutePath workingDirectory,
            DistributedContentStoreSettings settings,
            IAbsFileSystem fileSystem,
            IFileCopier <T> fileCopier,
            IFileExistenceChecker <T> fileExistenceChecker,
            ICopyRequester copyRequester,
            IPathTransformer <T> pathTransformer,
            IContentLocationStore contentLocationStore)
        {
            Contract.Requires(settings != null);
            Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);

            _settings                   = settings;
            _tempFolderForCopies        = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
            _remoteFileCopier           = fileCopier;
            _remoteFileExistenceChecker = fileExistenceChecker;
            _copyRequester              = copyRequester;
            _contentLocationStore       = contentLocationStore;
            _pathTransformer            = pathTransformer;
            _fileSystem                 = fileSystem;

            _workingDirectory = _tempFolderForCopies.Path;

            // TODO: Use hashers from IContentStoreInternal instead?
            _hashers = HashInfoLookup.CreateAll();

            _ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
            _proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
            _retryIntervals      = settings.RetryIntervalForCopies;

            _timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies;
        }
コード例 #6
0
        public Task SimpleBlobPutAndGetTest()
        {
            return(RunTest(async(context, service, iteration) =>
            {
                // First heartbeat lets the service know its master, so it's willing to process requests
                await service.OnRoleUpdatedAsync(context, Role.Master);

                var machineId = new MachineId(0);
                var data = ThreadSafeRandom.GetBytes((int)100);
                var contentHash = HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(data);

                var putResponse = await service.PutBlobAsync(new PutBlobRequest()
                {
                    ContextId = Guid.NewGuid().ToString(),
                    ContentHash = contentHash,
                    Blob = data
                });

                putResponse.Succeeded.Should().BeTrue();

                var getResponse = await service.GetBlobAsync(new GetBlobRequest()
                {
                    ContextId = Guid.NewGuid().ToString(),
                    ContentHash = contentHash,
                });

                getResponse.Succeeded.Should().BeTrue();
                getResponse.Blob.Should().NotBeNull();
                getResponse.Blob.Should().BeEquivalentTo(data);
            }));
        }
コード例 #7
0
        public void CreateContentHasher()
        {
            var hashInfo = HashInfoLookup.Find(_hasher.Info.HashType);

            // Warm-up session.
            PerfTestCase();

            var stopwatch = Stopwatch.StartNew();

            for (var i = 0; i < Iterations; i++)
            {
                PerfTestCase();
            }

            stopwatch.Stop();

            var rate = (long)(Iterations / stopwatch.Elapsed.TotalSeconds);
            var name = GetType().Name + "." + nameof(CreateContentHasher);

            _resultsFixture.AddResults(Output, name, rate, "items/sec", Iterations);

            void PerfTestCase()
            {
                using (var hasher = hashInfo.CreateContentHasher())
                {
                    hasher.Info.HashType.Should().Be(_hasher.Info.HashType);
                }
            }
        }
コード例 #8
0
        public static async Task <PutResult> PutRandomFileAsync(
            this IContentSession session,
            Context context,
            IAbsFileSystem fileSystem,
            HashType hashType,
            bool provideHash,
            long size,
            CancellationToken ct)
        {
            Contract.RequiresNotNull(session);
            Contract.RequiresNotNull(context);
            Contract.RequiresNotNull(fileSystem);

            using (var directory = new DisposableDirectory(fileSystem))
            {
                var c = context.CreateNested();

                // TODO: Fix this to work with size > int.Max (bug 1365340)
                var data = ThreadSafeRandom.GetBytes((int)size);
                var path = directory.CreateRandomFileName();
                fileSystem.WriteAllBytes(path, data);

                if (!provideHash)
                {
                    return(await session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct).ConfigureAwait(false));
                }

                var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);
                return(await session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct).ConfigureAwait(false));
            }
        }
コード例 #9
0
        /// <summary>
        ///     Put a randomly-sized content from a file into the store.
        /// </summary>
        public static Task <PutResult> PutRandomFileAsync(
            this IContentSession session,
            Context context,
            IAbsFileSystem fileSystem,
            AbsolutePath path,
            HashType hashType,
            bool provideHash,
            long size,
            CancellationToken ct)
        {
            Contract.RequiresNotNull(session);
            Contract.RequiresNotNull(context);
            Contract.RequiresNotNull(fileSystem);

            var c = context.CreateNested(nameof(ContentSessionExtensions));

            // TODO: Fix this to work with size > int.Max (bug 1365340)
            var data = ThreadSafeRandom.GetBytes((int)size);

            fileSystem.WriteAllBytes(path, data);

            if (!provideHash)
            {
                return(session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct));
            }

            var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);

            return(session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct));
        }
コード例 #10
0
        public static async Task <PutResult> PutRandomAsync(
            this IContentSession session,
            Context context,
            HashType hashType,
            bool provideHash,
            long size,
            CancellationToken ct)
        {
            Contract.RequiresNotNull(session);
            Contract.RequiresNotNull(context);

            var c = context.CreateNested();

            // TODO: Fix this to work with size > int.Max (bug 1365340)
            var data = ThreadSafeRandom.GetBytes((int)size);

            using (var stream = new MemoryStream(data))
            {
                if (!provideHash)
                {
                    return(await session.PutStreamAsync(c, hashType, stream, ct).ConfigureAwait(false));
                }

                var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);
                return(await session.PutStreamAsync(c, hash, stream, ct).ConfigureAwait(false));
            }
        }
コード例 #11
0
        public Task HandlesMultipleHashTypes()
        {
            var context = new Context(Logger);

            return(TestContentDirectory(context, async contentDirectory =>
            {
                using (var sha1Hasher = HashInfoLookup.Find(HashType.SHA1).CreateContentHasher())
                {
                    using (var sha256Hasher = HashInfoLookup.Find(HashType.SHA256).CreateContentHasher())
                    {
                        byte[] content = ThreadSafeRandom.GetBytes(DefaultFileSize);
                        ContentHash sha1ContentHash = sha1Hasher.GetContentHash(content);
                        ContentHash sha256ContentHash = sha256Hasher.GetContentHash(content);

                        await contentDirectory.UpdateAsync(sha1ContentHash, true, MemoryClock, info =>
                        {
                            Assert.Null(info);
                            return Task.FromResult(new ContentFileInfo(MemoryClock, 1, content.Length));
                        });

                        await contentDirectory.UpdateAsync(sha256ContentHash, true, MemoryClock, info =>
                        {
                            Assert.Null(info);
                            return Task.FromResult(new ContentFileInfo(MemoryClock, 2, content.Length));
                        });

                        var hashes = (await contentDirectory.EnumerateContentHashesAsync()).ToList();
                        hashes.Contains(sha1ContentHash).Should().BeTrue();
                        hashes.Contains(sha256ContentHash).Should().BeTrue();
                    }
                }
            }));
        }
コード例 #12
0
        public ColdStorage(IAbsFileSystem fileSystem, ColdStorageSettings coldStorageSettings, DistributedContentCopier distributedContentCopier)
        {
            _fileSystem = fileSystem;
            _copier     = distributedContentCopier;

            _rootPath = coldStorageSettings.GetAbsoulutePath();

            ConfigurationModel configurationModel
                = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(coldStorageSettings.CacheSizeQuotaString !)));

            ContentStoreSettings contentStoreSettings = FromColdStorageSettings(coldStorageSettings);

            _store = new FileSystemContentStore(fileSystem, SystemClock.Instance, _rootPath, configurationModel, null, contentStoreSettings, null);

            HashType hashType;

            if (!Enum.TryParse <HashType>(coldStorageSettings.ConsistentHashingHashType, true, out hashType))
            {
                hashType = HashType.SHA256;
            }
            _contentHasher = HashInfoLookup.GetContentHasher(hashType);

            _copiesQuantity    = coldStorageSettings.ConsistentHashingCopiesQuantity;
            _maxParallelPlaces = coldStorageSettings.MaxParallelPlaces;

            // Starts empty and is created during the first update
            _ring = new RingNode[0];
        }
コード例 #13
0
        public void ToHexCorrect(HashType hashType, string expected)
        {
            var hashLength = HashInfoLookup.Find(hashType).ByteLength;
            var hash       = new ContentHash(hashType, Enumerable.Range(0, hashLength).Select(i => (byte)i).ToArray());

            Assert.Equal(expected, hash.ToHex());
        }
コード例 #14
0
        public async Task HashBytesOfVariousSizes(int size)
        {
            using var fileSystem    = new PassThroughFileSystem();
            using var tempDirectory = new DisposableDirectory(fileSystem);

            var content = ThreadSafeRandom.GetBytes(size);
            var path    = tempDirectory.Path / "1.txt";

            fileSystem.WriteAllBytes(path, content);

            var hashType      = HashType.Vso0;
            var contentHasher = HashInfoLookup.GetContentHasher(hashType);

            var h1 = contentHasher.GetContentHash(content);
            var h2 = CalculateHashWithMemoryMappedFile(fileSystem, path, hashType);

            Assert.Equal(h1, h2);

#if NET_COREAPP
            h2 = contentHasher.GetContentHash(content.AsSpan());
            Assert.Equal(h1, h2);
#endif

            using var memoryStream = new MemoryStream(content);
            var h3 = await contentHasher.GetContentHashAsync(memoryStream);

            Assert.Equal(h1, h3);

            // Using an old style hashing to make sure it works the same as the new and optimized version.
            using var fileStream = fileSystem.OpenForHashing(path);
            var h4 = await contentHasher.GetContentHashAsync(fileStream);

            Assert.Equal(h1, h4);
        }
コード例 #15
0
ファイル: VsoHashInfoTests.cs プロジェクト: kittinap/kunnjae
 public void VsoContentHasherAlgorithmUsesVsoHashInfo()
 {
     using (var hasher = HashInfoLookup.Find(HashType.Vso0).CreateContentHasher())
     {
         Assert.Equal(VsoHashInfo.Instance.ByteLength, hasher.Info.ByteLength);
         Assert.Equal(VsoHashInfo.Instance.HashType.ToString(), hasher.Info.Name);
         Assert.Equal(VsoHashInfo.Instance.HashType, hasher.Info.HashType);
     }
 }
コード例 #16
0
        private static IContentHasher GetContentHasher(HashType hashType)
        {
            if (hashType == HashInfo.HashType || hashType == HashType.Unknown)
            {
                return(s_hasher);
            }

            return(s_contentHasherByHashType.GetOrAdd(hashType, (_) => HashInfoLookup.Find(hashType).CreateContentHasher()));
        }
コード例 #17
0
        /// <nodoc />
        internal static async Task <DedupNode> GetDedupNodeFromFileAsync(HashType hashType, string path)
        {
            var contentHasher = (DedupContentHasher <DedupNodeOrChunkHashAlgorithm>)HashInfoLookup.GetContentHasher(hashType);

            using (var stream = FileStreamUtility.OpenFileStreamForAsync(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
            {
                return(await contentHasher.HashContentAndGetDedupNodeAsync(stream));
            }
        }
コード例 #18
0
        public void ToHashBytesArray(HashType hashType)
        {
            var length      = HashInfoLookup.Find(hashType).ByteLength;
            var bytes       = Enumerable.Range(0, length).Select(i => (byte)i).ToArray();
            var contentHash = new ContentHash(hashType, bytes);
            var exported    = contentHash.ToHashByteArray();

            Assert.Equal(bytes, exported);
        }
コード例 #19
0
ファイル: BlobContentSession.cs プロジェクト: socat/BuildXL
        /// <inheritdoc />
        public async Task <PutResult> PutFileAsync(
            Context context,
            HashType hashType,
            AbsolutePath path,
            FileRealizationMode realizationMode,
            CancellationToken cts,
            UrgencyHint urgencyHint)
        {
            if (hashType != RequiredHashType)
            {
                return(new PutResult(
                           new ContentHash(hashType),
                           $"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{hashType}'."));
            }

            try
            {
                long        contentSize;
                ContentHash contentHash;
                using (var hashingStream = new FileStream(
                           path.Path,
                           FileMode.Open,
                           FileAccess.Read,
                           FileShare.Read | FileShare.Delete,
                           StreamBufferSize))
                {
                    contentSize = hashingStream.Length;
                    contentHash = await HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHashAsync(hashingStream).ConfigureAwait(false);
                }

                using (var streamToPut = FileStreamUtils.OpenFileStreamForAsync(
                           path.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
                {
                    BoolResult putSucceeded = await PutLazyStreamAsync(
                        context,
                        contentHash,
                        streamToPut,
                        cts,
                        urgencyHint).ConfigureAwait(false);

                    if (!putSucceeded.Succeeded)
                    {
                        return(new PutResult(
                                   putSucceeded,
                                   contentHash,
                                   $"Failed to add a BlobStore reference to content with hash=[{contentHash}]"));
                    }
                }

                return(new PutResult(contentHash, contentSize));
            }
            catch (Exception e)
            {
                return(new PutResult(e, new ContentHash(hashType)));
            }
        }
コード例 #20
0
        private ContentHash ComputeStartedCopyHash(ContentHash hash)
        {
            var murmurHash = MurmurHash3.Create(hash.ToByteArray(), _startedCopyHashSeed);

            var hashLength = HashInfoLookup.Find(_hashType).ByteLength;
            var buffer = murmurHash.ToByteArray();
            Array.Resize(ref buffer, hashLength);

            return new ContentHash(_hashType, buffer);
        }
コード例 #21
0
ファイル: CopyFileTo.cs プロジェクト: rustedwizard/BuildXL
        internal void CopyFileTo(
            [Required, Description("Machine to copy to")] string host,
            [Required, Description("Path to source file")] string sourcePath,
            [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified")] string grpcPortFileName,
            [Description("The GRPC port"), DefaultValue(0)] int grpcPort)
        {
            Initialize();

            var context          = new Context(_logger);
            var operationContext = new OperationContext(context, CancellationToken.None);
            var retryPolicy      = RetryPolicyFactory.GetLinearPolicy(ex => ex is ClientCanRetryException, (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds));

            if (grpcPort == 0)
            {
                grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
            }

            var hasher = HashInfoLookup.GetContentHasher(HashType.MD5);
            var bytes  = File.ReadAllBytes(sourcePath);
            var hash   = hasher.GetContentHash(bytes);

            try
            {
                var path = new AbsolutePath(sourcePath);
                using Stream stream = File.OpenRead(path.Path);

                var config = new GrpcCopyClientConfiguration();
                using var clientCache = new GrpcCopyClientCache(context, new GrpcCopyClientCacheConfiguration()
                {
                    GrpcCopyClientConfiguration = config
                });

                var copyFileResult = clientCache.UseAsync(operationContext, host, grpcPort, (nestedContext, rpcClient) =>
                {
                    return(retryPolicy.ExecuteAsync(
                               () => rpcClient.PushFileAsync(nestedContext, hash, stream, new CopyOptions(bandwidthConfiguration: null)),
                               _cancellationToken));
                }).GetAwaiter().GetResult();

                if (!copyFileResult.Succeeded)
                {
                    _tracer.Error(context, $"{copyFileResult}");
                    throw new CacheException(copyFileResult.ErrorMessage);
                }
                else
                {
                    _tracer.Info(context, $"Copy of {sourcePath} was successful");
                }
            }
            catch (Exception ex)
            {
                throw new CacheException(ex.ToString());
            }
        }
コード例 #22
0
ファイル: FileContentInfo.cs プロジェクト: kittinap/kunnjae
        public static bool IsValidLength(long length, ContentHash hash)
        {
            if (length < 0)
            {
                return(false);
            }

            ContentHash emptyHash = HashInfoLookup.Find(hash.HashType).EmptyHash;

            return(length == 0 ? (hash == emptyHash) : (hash != emptyHash));
        }
コード例 #23
0
        /// <inheritdoc />
        protected override async Task <PutResult> PutStreamCoreAsync(
            OperationContext context,
            HashType hashType,
            Stream stream,
            UrgencyHint urgencyHint,
            Counter retryCounter)
        {
            if (hashType != RequiredHashType)
            {
                return(new PutResult(
                           new ContentHash(hashType),
                           $"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{hashType}'."));
            }

            try
            {
                StreamWithLength streamToPut;

                // Can't assume we've been given a seekable stream.
                if (stream.CanSeek)
                {
                    streamToPut = stream.AssertHasLength();
                }
                else
                {
                    streamToPut = await CreateSeekableStreamAsync(context, stream);
                }

                using (streamToPut)
                {
                    Contract.Assert(streamToPut.Stream.CanSeek);
                    long contentSize = streamToPut.Length;
                    var  contentHash = await HashInfoLookup.Find(hashType)
                                       .CreateContentHasher()
                                       .GetContentHashAsync(streamToPut)
                                       .ConfigureAwait(false);

                    streamToPut.Stream.Seek(0, SeekOrigin.Begin);
                    var putResult =
                        await PutLazyStreamAsync(context, contentHash, streamToPut, urgencyHint).ConfigureAwait(false);

                    if (!putResult.Succeeded)
                    {
                        return(new PutResult(putResult, contentHash, $"Failed to add a BlobStore reference to content with hash=[{contentHash}]"));
                    }

                    return(new PutResult(contentHash, contentSize));
                }
            }
            catch (Exception e)
            {
                return(new PutResult(e, new ContentHash(hashType)));
            }
        }
コード例 #24
0
        public void Indexer(HashType hashType)
        {
            var length      = HashInfoLookup.Find(hashType).ByteLength;
            var bytes       = Enumerable.Range(0, length).Select(i => (byte)i).ToArray();
            var contentHash = new ContentHash(hashType, bytes);

            foreach (var i in Enumerable.Range(0, length))
            {
                var b = contentHash[i];
                Assert.Equal((byte)i, b);
            }
        }
コード例 #25
0
        public static bool IsValidLength(long length, ContentHash hash)
        {
            if (length < 0 ||
                length > LengthAndExistence.MaxSupportedLength ||
                hash.IsSpecialValue())
            {
                return(false);
            }

            ContentHash emptyHash = HashInfoLookup.Find(hash.HashType).EmptyHash;

            return(length == 0 ? (hash == emptyHash) : (hash != emptyHash));
        }
コード例 #26
0
        public void EqualContentHashRoundTripViaHexString(HashType hashType)
        {
            var h1  = ContentHash.Random(hashType);
            var hex = h1.ToHex();

            var hashInfo = HashInfoLookup.Find(hashType);
            var buffer   = new byte[hashInfo.ByteLength];

            var sb = HexUtilities.HexToBytes(hex, buffer);
            var h2 = new ContentHash(hashType, sb);

            Assert.Equal(h1, h2);
        }
コード例 #27
0
        public void RandomValue(HashType hashType)
        {
            var v = ContentHash.Random(hashType);

            Assert.Equal(hashType, v.HashType);

            var hashInfo = HashInfoLookup.Find(hashType);

            if (hashInfo is TaggedHashInfo taggedHashInfo)
            {
                Assert.Equal(v[hashInfo.ByteLength - 1], taggedHashInfo.AlgorithmId);
            }
        }
コード例 #28
0
        private async Task <bool> ValidateNameHashesMatchContentHashesAsync(Context context)
        {
            int mismatchedParentDirectoryCount = 0;
            int mismatchedContentHashCount     = 0;

            _tracer.Always(context, "Validating local CAS content hashes...");
            await TaskUtilities.SafeWhenAll(_enumerateBlobPathsFromDisk().Select(
                                                async blobPath =>
            {
                var contentFile = blobPath.FullPath;
                if (!contentFile.FileName.StartsWith(contentFile.GetParent().FileName, StringComparison.OrdinalIgnoreCase))
                {
                    mismatchedParentDirectoryCount++;

                    _tracer.Debug(
                        context,
                        $"The first {FileSystemContentStoreInternal.HashDirectoryNameLength} characters of the name of content file at {contentFile}" +
                        $" do not match the name of its parent directory {contentFile.GetParent().FileName}.");
                }

                if (!FileSystemContentStoreInternal.TryGetHashFromPath(context, _tracer, contentFile, out var hashFromPath))
                {
                    _tracer.Debug(
                        context,
                        $"The path '{contentFile}' does not contain a well-known hash name.");
                    return;
                }

                var hasher = HashInfoLookup.GetContentHasher(hashFromPath.HashType);
                ContentHash hashFromContents;
                using (var contentStream = _fileSystem.Open(
                           contentFile, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashingExtensions.HashStreamBufferSize))
                {
                    hashFromContents = await hasher.GetContentHashAsync(contentStream);
                }

                if (hashFromContents != hashFromPath)
                {
                    mismatchedContentHashCount++;

                    _tracer.Debug(
                        context,
                        $"Content at {contentFile} content hash {hashFromContents.ToShortString()} did not match expected value of {hashFromPath.ToShortString()}.");
                }
            }));

            _tracer.Always(context, $"{mismatchedParentDirectoryCount} mismatches between content file name and parent directory.");
            _tracer.Always(context, $"{mismatchedContentHashCount} mismatches between content file name and file contents.");

            return(mismatchedContentHashCount == 0 && mismatchedParentDirectoryCount == 0);
        }
コード例 #29
0
        public static async Task <PutResult> PutContentAsync(
            this IContentSession session, Context context, string content)
        {
            var c = context.CreateNested();

            var data     = Encoding.UTF8.GetBytes(content);
            var hashType = HashType.SHA256;

            using (var stream = new MemoryStream(data))
            {
                var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);
                return(await session.PutStreamAsync(c, hash, stream, CancellationToken.None).ConfigureAwait(false));
            }
        }
コード例 #30
0
        public Task TestColdStorageWithBulkFunction()
        {
            return(RunTestAsync(async(context, store, directory) =>
            {
                var originalPath = directory.Path / "original.txt";
                var fileContents = GetRandomFileContents();

                // Build destination IContentSession
                DisposableDirectory sessionDirectory = new DisposableDirectory(FileSystem);
                ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("10MB")));
                FileSystemContentStore destination = new FileSystemContentStore(FileSystem, SystemClock.Instance, sessionDirectory.Path, configurationModel);
                _ = await destination.StartupAsync(context);
                IContentSession contentSession = destination.CreateSession(context, "test_session", BuildXL.Cache.ContentStore.Interfaces.Stores.ImplicitPin.None).Session;
                _ = await contentSession.StartupAsync(context);

                // Create the file and hardlink it into the cache.
                FileSystem.WriteAllText(originalPath, fileContents);

                var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
                var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));

                await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();

                FileSystem.DeleteFile(originalPath);
                FileSystem.FileExists(originalPath).Should().Be(false);

                ContentHashWithPath contentHashWithPath = new ContentHashWithPath(contentHash, originalPath);
                List <ContentHashWithPath> listFile = new List <ContentHashWithPath>();
                listFile.Add(contentHashWithPath);

                // Hardlink back to original location trying to replace existing.
                var copyTask = await store.FetchThenPutBulkAsync(
                    context,
                    listFile,
                    contentSession);

                await copyTask.ToLookupAwait(r => { return r.Item.Succeeded; });

                FileSystem.FileExists(originalPath).Should().Be(false);

                // The file is in the destination.
                await contentSession.PlaceFileAsync(context, contentHash, originalPath, FileAccessMode.Write, FileReplacementMode.FailIfExists, FileRealizationMode.Copy, CancellationToken.None).ShouldBeSuccess();
                FileSystem.FileExists(originalPath).Should().Be(true);
                FileSystem.ReadAllText(originalPath).Should().Be(fileContents);

                _ = await contentSession.ShutdownAsync(context);
                _ = await destination.ShutdownAsync(context);
            }));
        }