public async Task CopyExistingFile() { await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) => { // Write a random file var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(sourcePath, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); // Copy the file out via GRPC var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); (await client.CopyFileAsync(_context, putResult.ContentHash, destinationPath, CancellationToken.None)).ShouldBeSuccess(); var copied = FileSystem.ReadAllBytes(destinationPath); // Compare original and copied files var originalHash = content.CalculateHash(DefaultHashType); var copiedHash = copied.CalculateHash(DefaultHashType); Assert.Equal(originalHash, copiedHash); }); }
public Task CopyFileRejectedIfTooMany(bool failFastIfServerBusy) { var failFastBandwidthConfiguration = new BandwidthConfiguration() { Interval = TimeSpan.FromSeconds(10), RequiredBytes = 10_000_000, FailFastIfServerIsBusy = failFastIfServerBusy, }; int numberOfFiles = 100; _copyToLimit = 1; return(RunTestCase(async(rootPath, session, client) => { // Add random files to the cache. var tasks = Enumerable.Range(1, numberOfFiles).Select(_ => putRandomFile()).ToList(); var hashes = await Task.WhenAll(tasks); var copyTasks = hashes.Select( hash => { var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); return client.CopyFileAsync( new OperationContext(_context), hash, destinationPath, new CopyOptions(failFastBandwidthConfiguration)); }); var results = await Task.WhenAll(copyTasks); if (failFastIfServerBusy) { // We're doing 100 simultaneous copies, at least some of them should fail, because we're not willing to wait for the response. var error = results.FirstOrDefault(r => !r.Succeeded); error.Should().NotBeNull("At least one copy operation should fail."); error !.ErrorMessage.Should().Contain("Copy limit of"); } else { // All operation should succeed! results.All(r => r.ShouldBeSuccess()).Should().BeTrue(); } async Task <ContentHash> putRandomFile() { var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(sourcePath, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); return putResult.ContentHash; } })); }
protected async Task PutRandomAndPinAsync(FileSystemContentStoreInternal store, int contentSize, PinContext pinContext) { PutResult putResult = await store.PutRandomAsync(Context, contentSize); putResult.ShouldBeSuccess(); PinResult pinResult = await store.PinAsync(Context, putResult.ContentHash, pinContext); pinResult.ShouldBeSuccess(); }
public Task AddImmediatelyDelete() { return(TestStore(Context, Clock, async(store) => { int contentSize = 10; PutResult putResult = await store.PutRandomAsync(Context, contentSize); putResult.ShouldBeSuccess(); DeleteResult deleteResult = await store.DeleteAsync(Context, putResult.ContentHash); deleteResult.ShouldBeSuccess(); deleteResult.EvictedSize.Should().Be(contentSize); deleteResult.PinnedSize.Should().Be(0); })); }
public async Task CheckExistingFile() { await RunTestCase(nameof(CheckExistingFile), async (rootPath, session, client) => { // Write a random file var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(sourcePath, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); // Check if file exists (await client.CheckFileExistsAsync(_context, putResult.ContentHash)).ShouldBeSuccess(); }); }
public async Task CopyExistingFile() { await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) => { // Write a random file to put into the cache var path = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(path, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, path, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); // Copy the file out via GRPC (await client.CopyFileAsync(_context, putResult.ContentHash, rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None)).ShouldBeSuccess(); }); }
public Task AddImmediatelyDelete() { return(TestStore(Context, Clock, async(store) => { store.QuotaKeeperSize().Should().Be(0); int contentSize = 10; PutResult putResult = await store.PutRandomAsync(Context, contentSize); putResult.ShouldBeSuccess(); DeleteResult deleteResult = await store.DeleteAsync(Context, putResult.ContentHash); deleteResult.ShouldBeSuccess(); deleteResult.ContentSize.Should().Be(contentSize); store.IsPinned(putResult.ContentHash).Should().BeFalse(); })); }
public async Task LargeStreamRoundtrip(bool provideHash, string sizeExpression) { _maxSize = "5GB".ToSize(); using (var directory = new DisposableDirectory(FileSystem)) { var path1 = directory.CreateRandomFileName(); var path2 = directory.CreateRandomFileName(); using (var fileStream = new FileStream(path1.Path, FileMode.Create, FileAccess.Write, FileShare.None)) { fileStream.SetLength(sizeExpression.ToSize()); } var contentHash1 = await FileSystem.CalculateHashAsync(path1, ContentHashType); using (var fileStream1 = File.OpenRead(path1.Path)) { await RunTestAsync(ImplicitPin.None, directory, async (context, session) => { // Stream content into the cache. PutResult r1 = provideHash ? await session.PutStreamAsync(context, contentHash1, fileStream1, Token) : await session.PutStreamAsync(context, ContentHashType, fileStream1, Token); r1.ShouldBeSuccess(); // Stream content back from the cache to a file. var r2 = await session.OpenStreamAsync(context, contentHash1, Token).ShouldBeSuccess(); using (r2.Stream) using (var fileStream2 = new FileStream(path2.Path, FileMode.Create, FileAccess.Write, FileShare.None)) { await r2.Stream.CopyToAsync(fileStream2); } // Verify received content is the same. var contentHash2 = await FileSystem.CalculateHashAsync(path2, ContentHashType); contentHash2.Should().Be(contentHash1); }); } } }
public Task CopyFailWithUnknownError() { return(RunTestCase(async(server, rootPath, session, client) => { // Write a random file var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(sourcePath, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); // Copy the file out via GRPC var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); // Injecting failure. server.GrpcContentServer.HandleRequestFailure = new Exception("Custom exception"); var copyFileResult = await client.CopyFileAsync(new OperationContext(_context), putResult.ContentHash, destinationPath, new CopyOptions(bandwidthConfiguration: null)); copyFileResult.ShouldBeError("Custom exception"); })); }
public async Task CopyToShouldNotCloseGivenStream() { await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) => { // Write a random file var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); var content = ThreadSafeRandom.GetBytes(FileSize); FileSystem.WriteAllBytes(sourcePath, content); // Put the random file PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None); putResult.ShouldBeSuccess(); // Copy the file out via GRPC var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString(); using (var destinationStream = await FileSystem.OpenAsync( destinationPath, FileAccess.ReadWrite, FileMode.CreateNew, FileShare.None, FileOptions.None, 1024)) { (await client.CopyToAsync(_context, putResult.ContentHash, destinationStream, CancellationToken.None)).ShouldBeSuccess(); // If the stream is not disposed, the following operation should not fail. destinationStream.Position.Should().BeGreaterThan(0); } var copied = FileSystem.ReadAllBytes(destinationPath); // Compare original and copied files var originalHash = content.CalculateHash(DefaultHashType); var copiedHash = copied.CalculateHash(DefaultHashType); Assert.Equal(originalHash, copiedHash); }); }
public async Task DoesNotRespectFileReplacementMode(FileReplacementMode requestedReplacementMode) { string scenario = nameof(DoesNotRespectFileReplacementMode) + requestedReplacementMode; var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var rootPath = directory.Path; var namedCacheRoots = new Dictionary <string, AbsolutePath> { { CacheName, rootPath } }; var grpcPort = PortExtensions.GetNextAvailablePort(); var grpcPortFileName = Guid.NewGuid().ToString(); var configuration = new ServiceConfiguration( namedCacheRoots, rootPath, MaxConnections, ServiceConfiguration.DefaultGracefulShutdownSeconds, grpcPort, grpcPortFileName); Func <AbsolutePath, IContentStore> contentStoreFactory = path => new FileSystemContentStore( FileSystem, SystemClock.Instance, rootPath, new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1MB")))); using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration))) { var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient"); await server.StartupAsync(context).ShouldBeSuccess(); var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort(); IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, port, scenario); await rpcClient.CreateSessionAsync( context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess(); ContentHash contentHash; using (var stream = new MemoryStream()) { PutResult putResult = await rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory : false); putResult.ShouldBeSuccess(); putResult.ContentSize.Should().Be(0); contentHash = putResult.ContentHash; } var tempPath = directory.CreateRandomFileName(); FileSystem.WriteAllBytes(tempPath, new byte[] {}); var placeResult = await rpcClient.PlaceFileAsync( context, contentHash, tempPath, FileAccessMode.ReadOnly, requestedReplacementMode, FileRealizationMode.Any); placeResult.Succeeded.Should().BeTrue(); (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess(); rpcClient.Dispose(); await server.ShutdownAsync(context).ShouldBeSuccess(); } } }