public async Task CopyExistingFile()
{
await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) =>
{
// Write a random file
var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(sourcePath, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
// Copy the file out via GRPC
var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
(await client.CopyFileAsync(_context, putResult.ContentHash, destinationPath, CancellationToken.None)).ShouldBeSuccess();
var copied = FileSystem.ReadAllBytes(destinationPath);
// Compare original and copied files
var originalHash = content.CalculateHash(DefaultHashType);
var copiedHash = copied.CalculateHash(DefaultHashType);
Assert.Equal(originalHash, copiedHash);
});
}
public Task CopyFileRejectedIfTooMany(bool failFastIfServerBusy)
{
var failFastBandwidthConfiguration = new BandwidthConfiguration()
{
Interval = TimeSpan.FromSeconds(10),
RequiredBytes = 10_000_000,
FailFastIfServerIsBusy = failFastIfServerBusy,
};
int numberOfFiles = 100;
_copyToLimit = 1;
return(RunTestCase(async(rootPath, session, client) =>
{
// Add random files to the cache.
var tasks = Enumerable.Range(1, numberOfFiles).Select(_ => putRandomFile()).ToList();
var hashes = await Task.WhenAll(tasks);
var copyTasks = hashes.Select(
hash =>
{
var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
return client.CopyFileAsync(
new OperationContext(_context),
hash,
destinationPath,
new CopyOptions(failFastBandwidthConfiguration));
});
var results = await Task.WhenAll(copyTasks);
if (failFastIfServerBusy)
{
// We're doing 100 simultaneous copies, at least some of them should fail, because we're not willing to wait for the response.
var error = results.FirstOrDefault(r => !r.Succeeded);
error.Should().NotBeNull("At least one copy operation should fail.");
error !.ErrorMessage.Should().Contain("Copy limit of");
}
else
{
// All operation should succeed!
results.All(r => r.ShouldBeSuccess()).Should().BeTrue();
}
async Task <ContentHash> putRandomFile()
{
var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(sourcePath, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
return putResult.ContentHash;
}
}));
}
protected async Task PutRandomAndPinAsync(FileSystemContentStoreInternal store, int contentSize, PinContext pinContext)
{
PutResult putResult = await store.PutRandomAsync(Context, contentSize);
putResult.ShouldBeSuccess();
PinResult pinResult = await store.PinAsync(Context, putResult.ContentHash, pinContext);
pinResult.ShouldBeSuccess();
}
public Task AddImmediatelyDelete()
{
return(TestStore(Context, Clock, async(store) =>
{
int contentSize = 10;
PutResult putResult = await store.PutRandomAsync(Context, contentSize);
putResult.ShouldBeSuccess();
DeleteResult deleteResult = await store.DeleteAsync(Context, putResult.ContentHash);
deleteResult.ShouldBeSuccess();
deleteResult.EvictedSize.Should().Be(contentSize);
deleteResult.PinnedSize.Should().Be(0);
}));
}
public async Task CheckExistingFile()
{
await RunTestCase(nameof(CheckExistingFile), async (rootPath, session, client) =>
{
// Write a random file
var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(sourcePath, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
// Check if file exists
(await client.CheckFileExistsAsync(_context, putResult.ContentHash)).ShouldBeSuccess();
});
}
public async Task CopyExistingFile()
{
await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) =>
{
// Write a random file to put into the cache
var path = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(path, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, path, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
// Copy the file out via GRPC
(await client.CopyFileAsync(_context, putResult.ContentHash, rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None)).ShouldBeSuccess();
});
}
public Task AddImmediatelyDelete()
{
return(TestStore(Context, Clock, async(store) =>
{
store.QuotaKeeperSize().Should().Be(0);
int contentSize = 10;
PutResult putResult = await store.PutRandomAsync(Context, contentSize);
putResult.ShouldBeSuccess();
DeleteResult deleteResult = await store.DeleteAsync(Context, putResult.ContentHash);
deleteResult.ShouldBeSuccess();
deleteResult.ContentSize.Should().Be(contentSize);
store.IsPinned(putResult.ContentHash).Should().BeFalse();
}));
}
public async Task LargeStreamRoundtrip(bool provideHash, string sizeExpression)
{
_maxSize = "5GB".ToSize();
using (var directory = new DisposableDirectory(FileSystem))
{
var path1 = directory.CreateRandomFileName();
var path2 = directory.CreateRandomFileName();
using (var fileStream = new FileStream(path1.Path, FileMode.Create, FileAccess.Write, FileShare.None))
{
fileStream.SetLength(sizeExpression.ToSize());
}
var contentHash1 = await FileSystem.CalculateHashAsync(path1, ContentHashType);
using (var fileStream1 = File.OpenRead(path1.Path))
{
await RunTestAsync(ImplicitPin.None, directory, async (context, session) =>
{
// Stream content into the cache.
PutResult r1 = provideHash
? await session.PutStreamAsync(context, contentHash1, fileStream1, Token)
: await session.PutStreamAsync(context, ContentHashType, fileStream1, Token);
r1.ShouldBeSuccess();
// Stream content back from the cache to a file.
var r2 = await session.OpenStreamAsync(context, contentHash1, Token).ShouldBeSuccess();
using (r2.Stream)
using (var fileStream2 = new FileStream(path2.Path, FileMode.Create, FileAccess.Write, FileShare.None))
{
await r2.Stream.CopyToAsync(fileStream2);
}
// Verify received content is the same.
var contentHash2 = await FileSystem.CalculateHashAsync(path2, ContentHashType);
contentHash2.Should().Be(contentHash1);
});
}
}
}
public Task CopyFailWithUnknownError()
{
return(RunTestCase(async(server, rootPath, session, client) =>
{
// Write a random file
var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(sourcePath, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
// Copy the file out via GRPC
var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
// Injecting failure.
server.GrpcContentServer.HandleRequestFailure = new Exception("Custom exception");
var copyFileResult = await client.CopyFileAsync(new OperationContext(_context), putResult.ContentHash, destinationPath, new CopyOptions(bandwidthConfiguration: null));
copyFileResult.ShouldBeError("Custom exception");
}));
}
public async Task CopyToShouldNotCloseGivenStream()
{
await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) =>
{
// Write a random file
var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(sourcePath, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
// Copy the file out via GRPC
var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
using (var destinationStream = await FileSystem.OpenAsync(
destinationPath,
FileAccess.ReadWrite,
FileMode.CreateNew,
FileShare.None,
FileOptions.None,
1024))
{
(await client.CopyToAsync(_context, putResult.ContentHash, destinationStream, CancellationToken.None)).ShouldBeSuccess();
// If the stream is not disposed, the following operation should not fail.
destinationStream.Position.Should().BeGreaterThan(0);
}
var copied = FileSystem.ReadAllBytes(destinationPath);
// Compare original and copied files
var originalHash = content.CalculateHash(DefaultHashType);
var copiedHash = copied.CalculateHash(DefaultHashType);
Assert.Equal(originalHash, copiedHash);
});
}
public async Task DoesNotRespectFileReplacementMode(FileReplacementMode requestedReplacementMode)
{
string scenario = nameof(DoesNotRespectFileReplacementMode) + requestedReplacementMode;
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1MB"))));
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, port, scenario);
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
ContentHash contentHash;
using (var stream = new MemoryStream())
{
PutResult putResult = await rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory : false);
putResult.ShouldBeSuccess();
putResult.ContentSize.Should().Be(0);
contentHash = putResult.ContentHash;
}
var tempPath = directory.CreateRandomFileName();
FileSystem.WriteAllBytes(tempPath, new byte[] {});
var placeResult = await rpcClient.PlaceFileAsync(
context, contentHash, tempPath, FileAccessMode.ReadOnly, requestedReplacementMode, FileRealizationMode.Any);
placeResult.Succeeded.Should().BeTrue();
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
rpcClient.Dispose();
await server.ShutdownAsync(context).ShouldBeSuccess();
}
}
}