public Task HandlesMultipleHashTypes() { var context = new Context(Logger); return(TestContentDirectory(context, async contentDirectory => { using (var sha1Hasher = HashInfoLookup.Find(HashType.SHA1).CreateContentHasher()) { using (var sha256Hasher = HashInfoLookup.Find(HashType.SHA256).CreateContentHasher()) { byte[] content = ThreadSafeRandom.GetBytes(DefaultFileSize); ContentHash sha1ContentHash = sha1Hasher.GetContentHash(content); ContentHash sha256ContentHash = sha256Hasher.GetContentHash(content); await contentDirectory.UpdateAsync(sha1ContentHash, true, MemoryClock, info => { Assert.Null(info); return Task.FromResult(new ContentFileInfo(MemoryClock, 1, content.Length)); }); await contentDirectory.UpdateAsync(sha256ContentHash, true, MemoryClock, info => { Assert.Null(info); return Task.FromResult(new ContentFileInfo(MemoryClock, 2, content.Length)); }); var hashes = (await contentDirectory.EnumerateContentHashesAsync()).ToList(); hashes.Contains(sha1ContentHash).Should().BeTrue(); hashes.Contains(sha256ContentHash).Should().BeTrue(); } } })); }
public async Task CopyExistingFile() { await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) => { // Write a random file var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(sourcePath, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); // Copy the file out via GRPC var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); (await client.CopyFileAsync(_context, putResult.ContentHash, destinationPath, CancellationToken.None)).ShouldBeSuccess(); var copied = FileSystem.ReadAllBytes(destinationPath); // Compare original and copied files var originalHash = content.CalculateHash(DefaultHashType); var copiedHash = copied.CalculateHash(DefaultHashType); Assert.Equal(originalHash, copiedHash); }); }
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!File.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } Stream s; if (FilesToCorrupt.ContainsKey(sourcePath)) { TestGlobal.Logger.Debug($"Corrupting file {sourcePath}"); s = new MemoryStream(ThreadSafeRandom.GetBytes((int)expectedContentSize)); } else { s = File.OpenRead(sourcePath.Path); } return(await s.CopyToAsync(destinationStream).ContinueWith((_) => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); }
public Task PutFileCreatesReadOnlyHardLink() { var context = new Context(Logger); return(TestStore(context, Clock, async store => { byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize); ContentHash contentHash = bytes.CalculateHash(ContentHashType); // Verify content doesn't exist yet in store Assert.False(await store.ContainsAsync(context, contentHash, null)); using (var tempDirectory = new DisposableDirectory(FileSystem)) { AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt"; FileSystem.WriteAllBytes(pathToContent, bytes); // Put the content into the store w/ hard link await store.PutFileAsync(context, pathToContent, FileRealizationMode.HardLink, ContentHashType, null).ShouldBeSuccess(); byte[] newBytes = ThreadSafeRandom.GetBytes(10); Assert.False(newBytes.SequenceEqual(bytes)); // Update content Action writeAction = () => FileSystem.WriteAllBytes(pathToContent, newBytes); writeAction.Should().Throw <UnauthorizedAccessException>(); } })); }
public Task PutFileWithExistingHashHardLinks() { var context = new Context(Logger); return(TestStore(context, Clock, async store => { using (var tempDirectory = new DisposableDirectory(FileSystem)) { var bytes = ThreadSafeRandom.GetBytes(100); AbsolutePath sourcePath = tempDirectory.CreateRandomFileName(); FileSystem.WriteAllBytes(sourcePath, bytes); await store.PutFileAsync(context, sourcePath, FileRealizationMode.HardLink, ContentHashType, null).ShouldBeSuccess(); AbsolutePath sourcePath2 = tempDirectory.CreateRandomFileName(); FileSystem.WriteAllBytes(sourcePath2, bytes); await store.PutFileAsync( context, sourcePath2, FileRealizationMode.HardLink, bytes.CalculateHash(ContentHashType), null).ShouldBeSuccess(); FileSystem.GetHardLinkCount(sourcePath2).Should().Be(3); } })); }
public Task PutFileMoveAppliesDenyWrites() { var context = new Context(Logger); return(TestStore(context, Clock, async store => { byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize); ContentHash contentHash = bytes.CalculateHash(ContentHashType); // Verify content doesn't exist yet in store Assert.False(await store.ContainsAsync(context, contentHash, null)); using (var tempDirectory = new DisposableDirectory(FileSystem)) { AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt"; FileSystem.WriteAllBytes(pathToContent, bytes); // Put the content into the store w/ hard link var r = await store.PutFileAsync( context, pathToContent, FileRealizationMode.Move, ContentHashType, null); // File should be deleted from original location FileSystem.FileExists(pathToContent).Should().BeFalse(); r.ContentHash.Should().Be(contentHash); var pathInStore = store.GetPrimaryPathFor(r.ContentHash); AbsFileSystemTests.VerifyThrowsOnOpenForWriteOfDenyWriteFile(FileSystem, pathInStore); } })); }
public async Task HashBytesOfVariousSizes(int size) { using var fileSystem = new PassThroughFileSystem(); using var tempDirectory = new DisposableDirectory(fileSystem); var content = ThreadSafeRandom.GetBytes(size); var path = tempDirectory.Path / "1.txt"; fileSystem.WriteAllBytes(path, content); var hashType = HashType.Vso0; var contentHasher = HashInfoLookup.GetContentHasher(hashType); var h1 = contentHasher.GetContentHash(content); var h2 = CalculateHashWithMemoryMappedFile(fileSystem, path, hashType); Assert.Equal(h1, h2); #if NET_COREAPP h2 = contentHasher.GetContentHash(content.AsSpan()); Assert.Equal(h1, h2); #endif using var memoryStream = new MemoryStream(content); var h3 = await contentHasher.GetContentHashAsync(memoryStream); Assert.Equal(h1, h3); // Using an old style hashing to make sure it works the same as the new and optimized version. using var fileStream = fileSystem.OpenForHashing(path); var h4 = await contentHasher.GetContentHashAsync(fileStream); Assert.Equal(h1, h4); }
public Task SomeLocalContentStoresCorrupt() { return(RunTestAsync(new Context(Logger), 3, async context => { var sessions = context.Sessions; // Insert random file in session 0 var randomBytes = ThreadSafeRandom.GetBytes(0x40); var putResult0 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes), Token).ShouldBeSuccess(); // Insert same file in session 1 var putResult1 = await sessions[1].PutStreamAsync(context.Context, HashType.Vso0, new MemoryStream(randomBytes), Token).ShouldBeSuccess(); putResult0.ContentHash.Should().Be(putResult1.ContentHash, "both puts of the same content should have the same hash."); // Delete file from local content store 0 var localContentPath = PathUtilities.GetContentPath(context.Directories[0].Path / "Root", putResult0.ContentHash); File.Delete(localContentPath.Path); context.FileCopier.FilesCopied.TryAdd(localContentPath, localContentPath); context.FileCopier.FileExistenceByReturnCode.GetOrAdd( localContentPath, (_) => { var queue = new ConcurrentQueue <FileExistenceResult.ResultCode>(); queue.Enqueue(FileExistenceResult.ResultCode.FileNotFound); return queue; }); // Query for file from session 2 await OpenStreamReturnsExpectedFile(sessions[2], context.Context, putResult0.ContentHash, randomBytes); })); }
public Task PinLargeSetsSucceeds() { return(RunTestAsync(new Context(Logger), 3, async context => { var sessions = context.Sessions; // Insert random file in session 0 List <ContentHash> contentHashes = new List <ContentHash>(); for (int i = 0; i < 250; i++) { var randomBytes1 = ThreadSafeRandom.GetBytes(0x40); var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token).ShouldBeSuccess(); contentHashes.Add(putResult1.ContentHash); } // Insert same file in session 1 IEnumerable <Task <Indexed <PinResult> > > pinResultTasks = await sessions[1].PinAsync(context, contentHashes, Token); foreach (var pinResultTask in pinResultTasks) { Assert.True((await pinResultTask).Item.Succeeded); } })); }
public Task RepeatedBlobIsSkipped() { return(RunTestAsync( new Context(Logger), 2, async context => { var sessions = context.Sessions; var session0 = context.GetDistributedSession(0); var redisStore0 = (RedisContentLocationStore)session0.ContentLocationStore; var session1 = context.GetDistributedSession(1); var redisStore1 = (RedisContentLocationStore)session1.ContentLocationStore; var file = ThreadSafeRandom.GetBytes(10); var fileString = Encoding.Default.GetString(file); await session0.PutContentAsync(context, fileString).ShouldBeSuccess(); var counters0 = redisStore0.GetCounters(context).ToDictionaryIntegral(); Assert.Equal(1, counters0["RedisContentLocationStore.BlobAdapter.PutBlob.Count"]); await session1.PutContentAsync(context, fileString).ShouldBeSuccess(); var counters1 = redisStore1.GetCounters(context).ToDictionaryIntegral(); Assert.Equal(1, counters1["RedisContentLocationStore.BlobAdapter.PutBlob.Count"]); Assert.Equal(1, counters1["RedisContentLocationStore.BlobAdapter.SkippedBlobs.Count"]); })); }
public async Task GetFilesFromDifferentReplicas() { await RunTestAsync(new Context(Logger), 3, async context => { var sessions = context.Sessions; // Insert random file in session 0 var randomBytes1 = ThreadSafeRandom.GetBytes(0x40); var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token); Assert.True(putResult1.Succeeded); // Insert random file in session 1 var randomBytes2 = ThreadSafeRandom.GetBytes(0x40); var putResult2 = await sessions[1].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes2), Token); Assert.True(putResult2.Succeeded); // Ensure both files are downloaded to session 2 await OpenStreamReturnsExpectedFile(sessions[2], context, putResult1.ContentHash, randomBytes1); await OpenStreamReturnsExpectedFile(sessions[2], context, putResult2.ContentHash, randomBytes2); // Query for other random file returns nothing var randomHash = ContentHash.Random(); var openStreamResult = await sessions[2].OpenStreamAsync(context, randomHash, Token); Assert.Equal(OpenStreamResult.ResultCode.ContentNotFound, openStreamResult.Code); Assert.Null(openStreamResult.Stream); }); Assert.True(true); }
public Task PutStreamParallelAdds() { const int PutSize = 30; return(TestStore(Context, Clock, async store => { var putTasks = Enumerable.Range(0, 20).Select(async i => { ContentHash hashFromPut; using (var dataStream = new MemoryStream(ThreadSafeRandom.GetBytes(PutSize))) { var r = await store.PutStreamAsync(Context, dataStream, ContentHashType, null).ShouldBeSuccess(); hashFromPut = r.ContentHash; Clock.Increment(); } return hashFromPut; }).ToArray(); ContentHash[] puthashes = await TaskUtilities.SafeWhenAll(putTasks); await store.SyncAsync(Context); var filesStillInCache = 0; foreach (var hash in puthashes) { filesStillInCache += (await store.ContainsAsync(Context, hash, null)) ? 1 : 0; } Assert.Equal(20, filesStillInCache); })); }
public async Task ContentDirectoryCreatedWhenMissing() { var context = new Context(Logger); var data1 = ThreadSafeRandom.GetBytes(MaxSizeHard / 3); using (var dataStream1 = new MemoryStream(data1)) { using (var testDirectory = new DisposableDirectory(FileSystem)) { var hash1 = new ContentHash(ContentHashType); await TestStore(context, _clock, testDirectory, async store => { store.DeleteContentDirectoryAfterDispose = true; hash1 = (await store.PutStreamAsync(context, dataStream1, ContentHashType, null)) .ContentHash; }); await TestStore(context, _clock, testDirectory, async store => { var r = await store.GetStatsAsync(context); r.CounterSet.GetIntegralWithNameLike("ReconstructCall").Should().Be(1); Assert.True(await store.ContainsAsync(context, hash1, null)); }); } } }
protected async Task <IReadOnlyList <KeyValuePair <ContentHash, ContentFileInfo> > > PopulateRandomInfo( IContentDirectory contentDirectory, Func <int, bool> shouldPopulateIteration = null) { var list = new List <KeyValuePair <ContentHash, ContentFileInfo> >(); for (var i = 0; i < DefaultInfoCount; i++) { byte[] content = ThreadSafeRandom.GetBytes(DefaultFileSize); using (var stream = new MemoryStream(content)) { ContentHash contentHash = await _hasher.GetContentHashAsync(stream); var newInfo = new ContentFileInfo(MemoryClock, content.Length); if (shouldPopulateIteration == null || shouldPopulateIteration(i)) { await contentDirectory.UpdateAsync(contentHash, false, MemoryClock, fileInfo => { Assert.Null(fileInfo); return(Task.FromResult(newInfo)); }); } list.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, newInfo)); } } return(list); }
public void CreateFromBytes() { const int count = 2000000; var hashType = _hasher.Info.HashType; var hashBytes = ThreadSafeRandom.GetBytes(_hasher.Info.ByteLength); PerfTest(); // Warm-up invocation. var stopwatch = Stopwatch.StartNew(); for (var i = 0; i < count; i++) { PerfTest(); } stopwatch.Stop(); var rate = (long)(count / stopwatch.Elapsed.TotalSeconds); var name = GetType().Name + "." + nameof(CreateFromBytes); _resultsFixture.AddResults(Output, name, rate, "items/sec", count); void PerfTest() { var hash = new ContentHash(hashType, hashBytes); DoSomethingWith(hash); } }
public async Task PinUpdatesExpiryWhenFoundRemote() { await RunTestAsync(new Context(Logger), 2, async context => { var sessions = context.Sessions; var session = context.GetDistributedSession(0); var redisStore = context.GetRedisStore(session); // Insert random file in session 0 with expiry of 30 minutes redisStore.ContentHashBumpTime = TimeSpan.FromMinutes(30); var randomBytes1 = ThreadSafeRandom.GetBytes(0x40); var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token); Assert.True(putResult1.Succeeded); // Insert same file in session 1 and update expiry to 1 hour redisStore.ContentHashBumpTime = TimeSpan.FromHours(1); IEnumerable <Task <Indexed <PinResult> > > pinResultTasks = await sessions[1].PinAsync(context, new[] { putResult1.ContentHash }, Token); Assert.True((await pinResultTasks.First()).Item.Succeeded); var expiry = await redisStore.GetContentHashExpiryAsync(new Context(Logger), putResult1.ContentHash, CancellationToken.None); Assert.True(expiry.HasValue); Assert.InRange(expiry.Value, redisStore.ContentHashBumpTime - TimeSpan.FromSeconds(15), redisStore.ContentHashBumpTime + TimeSpan.FromSeconds(15)); }); }
private void Run(string method, int contentSize, int iterations = Iterations) { var content = ThreadSafeRandom.GetBytes(contentSize); PerfTestCase(); // warm up var stopwatch = Stopwatch.StartNew(); for (var i = 0; i < iterations; i++) { PerfTestCase(); } stopwatch.Stop(); var rate = (long)(iterations / stopwatch.Elapsed.TotalSeconds); var name = GetType().Name + "." + method; _resultsFixture.AddResults(Output, name, rate, "items/sec", iterations); void PerfTestCase() { var x = _hasher.GetContentHash(content); if (x.ByteLength != _hasher.Info.ByteLength) { throw new InvalidOperationException("unexpected hash length"); } } }
/// <summary> /// Put a randomly-sized content from a file into the store. /// </summary> public static Task <PutResult> PutRandomFileAsync( this IContentSession session, Context context, IAbsFileSystem fileSystem, AbsolutePath path, HashType hashType, bool provideHash, long size, CancellationToken ct) { Contract.RequiresNotNull(session); Contract.RequiresNotNull(context); Contract.RequiresNotNull(fileSystem); var c = context.CreateNested(nameof(ContentSessionExtensions)); // TODO: Fix this to work with size > int.Max (bug 1365340) var data = ThreadSafeRandom.GetBytes((int)size); fileSystem.WriteAllBytes(path, data); if (!provideHash) { return(session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct)); } var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct)); }
public Task PutFileExceedQuotaSucceeds() { var context = new Context(Logger); return(TestStore(context, Clock, async store => { int size = MaxSizeHard + ValueSize; byte[] bytes = ThreadSafeRandom.GetBytes(size); using (var tempDirectory = new DisposableDirectory(FileSystem)) { AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt"; FileSystem.WriteAllBytes(pathToContent, bytes); ContentHash hash; using (var pinContext = store.CreatePinContext()) { var r = await store.PutFileAsync( context, pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext)).ShouldBeSuccess(); hash = r.ContentHash; Clock.Increment(); } Assert.True(await store.ContainsAsync(context, hash, null)); // Sync to allow calibration to occur. await store.SyncAsync(context); var currentQuota = await LoadElasticQuotaAsync(store.RootPathForTest); Assert.NotNull(currentQuota); // Calibration should adjust the quota. Assert.True(size < currentQuota.Quota.Hard); } })); }
public Task RepeatedBlobIsSkipped() { ConfigureWithOneMasterAndSmallBlobs(); return(RunTestAsync( new Context(Logger), 2, async context => { var sessions = context.Sessions; var session0 = context.GetDistributedSession(0); var redisStore0 = context.GetRedisGlobalStore(0); var session1 = context.GetDistributedSession(1); var redisStore1 = context.GetRedisGlobalStore(1); var file = ThreadSafeRandom.GetBytes(10); var fileString = Encoding.Default.GetString(file); await session0.PutContentAsync(context, fileString).ShouldBeSuccess(); Assert.Equal(1, redisStore0.Counters[GlobalStoreCounters.PutBlob].Value); await session1.PutContentAsync(context, fileString).ShouldBeSuccess(); Assert.Equal(1, redisStore1.Counters[GlobalStoreCounters.PutBlob].Value); Assert.Equal(1, redisStore1.BlobAdapter.Counters[RedisBlobAdapter.RedisBlobAdapterCounters.SkippedBlobs].Value); })); }
public async Task PushIsRejectedForTheSameHash() { await Task.Yield(); int numberOfPushes = 100; await RunTestCase(async (rootPath, session, client) => { var bytes = ThreadSafeRandom.GetBytes(1 + 42); var input = Enumerable.Range(1, numberOfPushes) .Select(data => (stream: new MemoryStream(bytes), hash: HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(bytes))) .ToList(); var pushTasks = input.Select( tpl => client.PushFileAsync( new OperationContext(_context), tpl.hash, tpl.stream, new CopyOptions(bandwidthConfiguration: null))).ToList(); var results = await Task.WhenAll(pushTasks); results.Any(r => r.Status == CopyResultCode.Rejected_OngoingCopy).Should().BeTrue(); var result = await client.PushFileAsync( new OperationContext(_context), input[0].hash, input[0].stream, new CopyOptions(bandwidthConfiguration: null)); result.Status.Should().Be(CopyResultCode.Rejected_ContentAvailableLocally); }); }
public static async Task <PutResult> PutRandomAsync( this IContentSession session, Context context, HashType hashType, bool provideHash, long size, CancellationToken ct) { Contract.RequiresNotNull(session); Contract.RequiresNotNull(context); var c = context.CreateNested(); // TODO: Fix this to work with size > int.Max (bug 1365340) var data = ThreadSafeRandom.GetBytes((int)size); using (var stream = new MemoryStream(data)) { if (!provideHash) { return(await session.PutStreamAsync(c, hashType, stream, ct).ConfigureAwait(false)); } var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(await session.PutStreamAsync(c, hash, stream, ct).ConfigureAwait(false)); } }
public Task PutFileHardLinkOverwritesSourceOnMismatchedExistingContentHash() { var context = new Context(Logger); return(TestStore(context, Clock, async store => { using (var tempDirectory = new DisposableDirectory(FileSystem)) { var bytes = ThreadSafeRandom.GetBytes(100); var contentHash = bytes.CalculateHash(ContentHashType); AbsolutePath sourcePath = tempDirectory.CreateRandomFileName(); FileSystem.WriteAllBytes(sourcePath, bytes); await store.PutFileAsync(context, sourcePath, FileRealizationMode.HardLink, ContentHashType, null).ShouldBeSuccess(); var bytes2 = ThreadSafeRandom.GetBytes(200); AbsolutePath sourcePath2 = tempDirectory.CreateRandomFileName(); FileSystem.WriteAllBytes(sourcePath2, bytes2); var result = await store.PutFileAsync(context, sourcePath2, FileRealizationMode.HardLink, contentHash, null); result.ContentHash.Should().Be(contentHash); using (StreamWithLength? stream = await FileSystem.OpenAsync( sourcePath2, FileAccess.Read, FileMode.Open, FileShare.Read)) { (await stream.Value.CalculateHashAsync(ContentHashType)).Should().Be(contentHash); } } })); }
public static async Task <PutResult> PutRandomFileAsync( this IContentSession session, Context context, IAbsFileSystem fileSystem, HashType hashType, bool provideHash, long size, CancellationToken ct) { Contract.RequiresNotNull(session); Contract.RequiresNotNull(context); Contract.RequiresNotNull(fileSystem); using (var directory = new DisposableDirectory(fileSystem)) { var c = context.CreateNested(); // TODO: Fix this to work with size > int.Max (bug 1365340) var data = ThreadSafeRandom.GetBytes((int)size); var path = directory.CreateRandomFileName(); fileSystem.WriteAllBytes(path, data); if (!provideHash) { return(await session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct).ConfigureAwait(false)); } var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data); return(await session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct).ConfigureAwait(false)); } }
public Task PutFilePins() { var context = new Context(Logger); return(TestStore(context, Clock, async store => { byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize); ContentHash contentHash = bytes.CalculateHash(ContentHashType); // Verify content doesn't exist yet in store Assert.False(await store.ContainsAsync(context, contentHash, null)); using (var tempDirectory = new DisposableDirectory(FileSystem)) { AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt"; FileSystem.WriteAllBytes(pathToContent, bytes); ContentHash hashFromPut; using (var pinContext = store.CreatePinContext()) { // Put the content into the store w/ hard link var r = await store.PutFileAsync( context, pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext)); hashFromPut = r.ContentHash; Clock.Increment(); await store.EnsureContentIsPinned(context, Clock, hashFromPut); Assert.True(pinContext.Contains(hashFromPut)); } await store.EnsureContentIsNotPinned(context, Clock, hashFromPut); } })); }
public Task SuccessivePutStreamsFullPinnedCacheSucceed() { return(TestStore(Context, Clock, async store => { using (var pinContext = store.CreatePinContext()) { var pinRequest = new PinRequest(pinContext); var r = await store.PutRandomAsync(Context, MaxSizeHard / 3).ShouldBeSuccess(); Clock.Increment(); Assert.True(await store.ContainsAsync(Context, r.ContentHash, pinRequest)); Clock.Increment(); r = await store.PutRandomAsync(Context, MaxSizeHard / 3); Clock.Increment(); Assert.True(await store.ContainsAsync(Context, r.ContentHash, pinRequest)); Clock.Increment(); var data = ThreadSafeRandom.GetBytes(MaxSizeHard / 2); using (var dataStream = new MemoryStream(data)) { await store.PutStreamAsync(Context, dataStream, ContentHashType, null).ShouldBeSuccess(); } await store.SyncAsync(Context); } })); }
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { FilesCopied.AddOrUpdate(destinationPath, p => path, (dest, prevPath) => overwrite ? path : prevPath); if (!File.Exists(path.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {path} doesn't exist.")); } if (File.Exists(destinationPath.Path)) { if (!overwrite) { return(new CopyFileResult( CopyFileResult.ResultCode.DestinationPathError, $"Destination file {destinationPath} exists but overwrite not specified.")); } } if (FilesToCorrupt.ContainsKey(path)) { TestGlobal.Logger.Debug($"Corrupting file {path}"); #pragma warning disable AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes await Task.Run( () => File.WriteAllBytes(destinationPath.Path, ThreadSafeRandom.GetBytes(150))); #pragma warning restore AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes } else { await Task.Run(() => File.Copy(path.Path, destinationPath.Path), cancellationToken); } return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length)); }
public Task RepeatedBlobIsSkipped() { ConfigureWithOneMasterAndSmallBlobs(); return(RunTestAsync( new Context(Logger), 2, async context => { var sessions = context.Sessions; var session0 = context.GetSession(0); var redisStore0 = context.GetRedisGlobalStore(0); var session1 = context.GetSession(1); var redisStore1 = context.GetRedisGlobalStore(1); var file = ThreadSafeRandom.GetBytes(10); var fileString = Encoding.Default.GetString(file); await session0.PutContentAsync(context, fileString).ShouldBeSuccess(); Assert.Equal(1, redisStore0.Counters[GlobalStoreCounters.PutBlob].Value); var result = await session1.PutContentAsync(context, fileString).ShouldBeSuccess(); Assert.Equal(1, redisStore1.Counters[GlobalStoreCounters.PutBlob].Value); var counters = redisStore1.GetBlobAdapter(result.ContentHash).GetCounters().ToDictionaryIntegral(); Assert.Equal(1, counters["SkippedBlobs.Count"]); })); }
public async Task EvictionAnnouncesHash() { bool batchProcessWasCalled = false; var nagleQueue = NagleQueue <ContentHash> .Create( hashes => { batchProcessWasCalled = true; return(Task.FromResult(42)); }, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(1), batchSize : 1); await TestStore( _context, _clock, async store => { var cas = store as IContentStoreInternal; var blobSize = BlobSizeToStartSoftPurging(2); using (var stream1 = new MemoryStream(ThreadSafeRandom.GetBytes(blobSize))) using (var stream2 = new MemoryStream(ThreadSafeRandom.GetBytes(blobSize))) { await cas.PutStreamAsync(_context, stream1, ContentHashType).ShouldBeSuccess(); _clock.Increment(); await cas.PutStreamAsync(_context, stream2, ContentHashType).ShouldBeSuccess(); _clock.Increment(); await store.SyncAsync(_context); } }, nagleQueue); batchProcessWasCalled.Should().BeTrue(); }
[Trait("Category", "QTestSkip")] // Skipped public async Task DeleteOneLinkWhileOneOtherLinkIsOpenReadOnlySharingFailsWithAppropriateError() { using (var testDirectory = new DisposableDirectory(FileSystem)) { var sourcePath = testDirectory.Path / @"source.txt"; var destinationPath1 = testDirectory.Path / @"destination1.txt"; var destinationPath2 = testDirectory.Path / @"destination2.txt"; FileSystem.WriteAllBytes(sourcePath, ThreadSafeRandom.GetBytes(10)); Assert.Equal(CreateHardLinkResult.Success, FileSystem.CreateHardLink(sourcePath, destinationPath1, true)); Assert.Equal(CreateHardLinkResult.Success, FileSystem.CreateHardLink(sourcePath, destinationPath2, true)); using (await FileSystem.OpenAsync(sourcePath, FileAccess.Read, FileMode.Open, ShareReadDelete)) using (await FileSystem.OpenAsync(destinationPath2, FileAccess.Read, FileMode.Open, ShareRead)) { Action a = () => FileSystem.DeleteFile(destinationPath1); var exception = Record.Exception(a); Assert.NotNull(exception); Assert.IsType <UnauthorizedAccessException>(exception); Assert.Contains( "The process cannot access the file because it is being used by another process.", exception.Message, StringComparison.OrdinalIgnoreCase); } } }