public Task HandlesMultipleHashTypes()
{
var context = new Context(Logger);
return(TestContentDirectory(context, async contentDirectory =>
{
using (var sha1Hasher = HashInfoLookup.Find(HashType.SHA1).CreateContentHasher())
{
using (var sha256Hasher = HashInfoLookup.Find(HashType.SHA256).CreateContentHasher())
{
byte[] content = ThreadSafeRandom.GetBytes(DefaultFileSize);
ContentHash sha1ContentHash = sha1Hasher.GetContentHash(content);
ContentHash sha256ContentHash = sha256Hasher.GetContentHash(content);
await contentDirectory.UpdateAsync(sha1ContentHash, true, MemoryClock, info =>
{
Assert.Null(info);
return Task.FromResult(new ContentFileInfo(MemoryClock, 1, content.Length));
});
await contentDirectory.UpdateAsync(sha256ContentHash, true, MemoryClock, info =>
{
Assert.Null(info);
return Task.FromResult(new ContentFileInfo(MemoryClock, 2, content.Length));
});
var hashes = (await contentDirectory.EnumerateContentHashesAsync()).ToList();
hashes.Contains(sha1ContentHash).Should().BeTrue();
hashes.Contains(sha256ContentHash).Should().BeTrue();
}
}
}));
}
public async Task CopyExistingFile()
{
await RunTestCase(nameof(CopyExistingFile), async (rootPath, session, client) =>
{
// Write a random file
var sourcePath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
var content = ThreadSafeRandom.GetBytes(FileSize);
FileSystem.WriteAllBytes(sourcePath, content);
// Put the random file
PutResult putResult = await session.PutFileAsync(_context, HashType.Vso0, sourcePath, FileRealizationMode.Any, CancellationToken.None);
putResult.ShouldBeSuccess();
// Copy the file out via GRPC
var destinationPath = rootPath / ThreadSafeRandom.Generator.Next().ToString();
(await client.CopyFileAsync(_context, putResult.ContentHash, destinationPath, CancellationToken.None)).ShouldBeSuccess();
var copied = FileSystem.ReadAllBytes(destinationPath);
// Compare original and copied files
var originalHash = content.CalculateHash(DefaultHashType);
var copiedHash = copied.CalculateHash(DefaultHashType);
Assert.Equal(originalHash, copiedHash);
});
}
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!File.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
Stream s;
if (FilesToCorrupt.ContainsKey(sourcePath))
{
TestGlobal.Logger.Debug($"Corrupting file {sourcePath}");
s = new MemoryStream(ThreadSafeRandom.GetBytes((int)expectedContentSize));
}
else
{
s = File.OpenRead(sourcePath.Path);
}
return(await s.CopyToAsync(destinationStream).ContinueWith((_) => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
public Task PutFileCreatesReadOnlyHardLink()
{
var context = new Context(Logger);
return(TestStore(context, Clock, async store =>
{
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
// Put the content into the store w/ hard link
await store.PutFileAsync(context, pathToContent, FileRealizationMode.HardLink, ContentHashType, null).ShouldBeSuccess();
byte[] newBytes = ThreadSafeRandom.GetBytes(10);
Assert.False(newBytes.SequenceEqual(bytes));
// Update content
Action writeAction = () => FileSystem.WriteAllBytes(pathToContent, newBytes);
writeAction.Should().Throw <UnauthorizedAccessException>();
}
}));
}
public Task PutFileWithExistingHashHardLinks()
{
var context = new Context(Logger);
return(TestStore(context, Clock, async store =>
{
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
var bytes = ThreadSafeRandom.GetBytes(100);
AbsolutePath sourcePath = tempDirectory.CreateRandomFileName();
FileSystem.WriteAllBytes(sourcePath, bytes);
await store.PutFileAsync(context, sourcePath, FileRealizationMode.HardLink, ContentHashType, null).ShouldBeSuccess();
AbsolutePath sourcePath2 = tempDirectory.CreateRandomFileName();
FileSystem.WriteAllBytes(sourcePath2, bytes);
await store.PutFileAsync(
context,
sourcePath2,
FileRealizationMode.HardLink,
bytes.CalculateHash(ContentHashType),
null).ShouldBeSuccess();
FileSystem.GetHardLinkCount(sourcePath2).Should().Be(3);
}
}));
}
public Task PutFileMoveAppliesDenyWrites()
{
var context = new Context(Logger);
return(TestStore(context, Clock, async store =>
{
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
// Put the content into the store w/ hard link
var r = await store.PutFileAsync(
context, pathToContent, FileRealizationMode.Move, ContentHashType, null);
// File should be deleted from original location
FileSystem.FileExists(pathToContent).Should().BeFalse();
r.ContentHash.Should().Be(contentHash);
var pathInStore = store.GetPrimaryPathFor(r.ContentHash);
AbsFileSystemTests.VerifyThrowsOnOpenForWriteOfDenyWriteFile(FileSystem, pathInStore);
}
}));
}
public async Task HashBytesOfVariousSizes(int size)
{
using var fileSystem = new PassThroughFileSystem();
using var tempDirectory = new DisposableDirectory(fileSystem);
var content = ThreadSafeRandom.GetBytes(size);
var path = tempDirectory.Path / "1.txt";
fileSystem.WriteAllBytes(path, content);
var hashType = HashType.Vso0;
var contentHasher = HashInfoLookup.GetContentHasher(hashType);
var h1 = contentHasher.GetContentHash(content);
var h2 = CalculateHashWithMemoryMappedFile(fileSystem, path, hashType);
Assert.Equal(h1, h2);
#if NET_COREAPP
h2 = contentHasher.GetContentHash(content.AsSpan());
Assert.Equal(h1, h2);
#endif
using var memoryStream = new MemoryStream(content);
var h3 = await contentHasher.GetContentHashAsync(memoryStream);
Assert.Equal(h1, h3);
// Using an old style hashing to make sure it works the same as the new and optimized version.
using var fileStream = fileSystem.OpenForHashing(path);
var h4 = await contentHasher.GetContentHashAsync(fileStream);
Assert.Equal(h1, h4);
}
public Task SomeLocalContentStoresCorrupt()
{
return(RunTestAsync(new Context(Logger), 3, async context =>
{
var sessions = context.Sessions;
// Insert random file in session 0
var randomBytes = ThreadSafeRandom.GetBytes(0x40);
var putResult0 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes), Token).ShouldBeSuccess();
// Insert same file in session 1
var putResult1 = await sessions[1].PutStreamAsync(context.Context, HashType.Vso0, new MemoryStream(randomBytes), Token).ShouldBeSuccess();
putResult0.ContentHash.Should().Be(putResult1.ContentHash, "both puts of the same content should have the same hash.");
// Delete file from local content store 0
var localContentPath = PathUtilities.GetContentPath(context.Directories[0].Path / "Root", putResult0.ContentHash);
File.Delete(localContentPath.Path);
context.FileCopier.FilesCopied.TryAdd(localContentPath, localContentPath);
context.FileCopier.FileExistenceByReturnCode.GetOrAdd(
localContentPath,
(_) =>
{
var queue = new ConcurrentQueue <FileExistenceResult.ResultCode>();
queue.Enqueue(FileExistenceResult.ResultCode.FileNotFound);
return queue;
});
// Query for file from session 2
await OpenStreamReturnsExpectedFile(sessions[2], context.Context, putResult0.ContentHash, randomBytes);
}));
}
public Task PinLargeSetsSucceeds()
{
return(RunTestAsync(new Context(Logger), 3, async context =>
{
var sessions = context.Sessions;
// Insert random file in session 0
List <ContentHash> contentHashes = new List <ContentHash>();
for (int i = 0; i < 250; i++)
{
var randomBytes1 = ThreadSafeRandom.GetBytes(0x40);
var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token).ShouldBeSuccess();
contentHashes.Add(putResult1.ContentHash);
}
// Insert same file in session 1
IEnumerable <Task <Indexed <PinResult> > > pinResultTasks = await sessions[1].PinAsync(context, contentHashes, Token);
foreach (var pinResultTask in pinResultTasks)
{
Assert.True((await pinResultTask).Item.Succeeded);
}
}));
}
public Task RepeatedBlobIsSkipped()
{
return(RunTestAsync(
new Context(Logger),
2,
async context =>
{
var sessions = context.Sessions;
var session0 = context.GetDistributedSession(0);
var redisStore0 = (RedisContentLocationStore)session0.ContentLocationStore;
var session1 = context.GetDistributedSession(1);
var redisStore1 = (RedisContentLocationStore)session1.ContentLocationStore;
var file = ThreadSafeRandom.GetBytes(10);
var fileString = Encoding.Default.GetString(file);
await session0.PutContentAsync(context, fileString).ShouldBeSuccess();
var counters0 = redisStore0.GetCounters(context).ToDictionaryIntegral();
Assert.Equal(1, counters0["RedisContentLocationStore.BlobAdapter.PutBlob.Count"]);
await session1.PutContentAsync(context, fileString).ShouldBeSuccess();
var counters1 = redisStore1.GetCounters(context).ToDictionaryIntegral();
Assert.Equal(1, counters1["RedisContentLocationStore.BlobAdapter.PutBlob.Count"]);
Assert.Equal(1, counters1["RedisContentLocationStore.BlobAdapter.SkippedBlobs.Count"]);
}));
}
public async Task GetFilesFromDifferentReplicas()
{
await RunTestAsync(new Context(Logger), 3, async context =>
{
var sessions = context.Sessions;
// Insert random file in session 0
var randomBytes1 = ThreadSafeRandom.GetBytes(0x40);
var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token);
Assert.True(putResult1.Succeeded);
// Insert random file in session 1
var randomBytes2 = ThreadSafeRandom.GetBytes(0x40);
var putResult2 = await sessions[1].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes2), Token);
Assert.True(putResult2.Succeeded);
// Ensure both files are downloaded to session 2
await OpenStreamReturnsExpectedFile(sessions[2], context, putResult1.ContentHash, randomBytes1);
await OpenStreamReturnsExpectedFile(sessions[2], context, putResult2.ContentHash, randomBytes2);
// Query for other random file returns nothing
var randomHash = ContentHash.Random();
var openStreamResult = await sessions[2].OpenStreamAsync(context, randomHash, Token);
Assert.Equal(OpenStreamResult.ResultCode.ContentNotFound, openStreamResult.Code);
Assert.Null(openStreamResult.Stream);
});
Assert.True(true);
}
public Task PutStreamParallelAdds()
{
const int PutSize = 30;
return(TestStore(Context, Clock, async store =>
{
var putTasks = Enumerable.Range(0, 20).Select(async i =>
{
ContentHash hashFromPut;
using (var dataStream = new MemoryStream(ThreadSafeRandom.GetBytes(PutSize)))
{
var r = await store.PutStreamAsync(Context, dataStream, ContentHashType, null).ShouldBeSuccess();
hashFromPut = r.ContentHash;
Clock.Increment();
}
return hashFromPut;
}).ToArray();
ContentHash[] puthashes = await TaskUtilities.SafeWhenAll(putTasks);
await store.SyncAsync(Context);
var filesStillInCache = 0;
foreach (var hash in puthashes)
{
filesStillInCache += (await store.ContainsAsync(Context, hash, null)) ? 1 : 0;
}
Assert.Equal(20, filesStillInCache);
}));
}
public async Task ContentDirectoryCreatedWhenMissing()
{
var context = new Context(Logger);
var data1 = ThreadSafeRandom.GetBytes(MaxSizeHard / 3);
using (var dataStream1 = new MemoryStream(data1))
{
using (var testDirectory = new DisposableDirectory(FileSystem))
{
var hash1 = new ContentHash(ContentHashType);
await TestStore(context, _clock, testDirectory, async store =>
{
store.DeleteContentDirectoryAfterDispose = true;
hash1 = (await store.PutStreamAsync(context, dataStream1, ContentHashType, null))
.ContentHash;
});
await TestStore(context, _clock, testDirectory, async store =>
{
var r = await store.GetStatsAsync(context);
r.CounterSet.GetIntegralWithNameLike("ReconstructCall").Should().Be(1);
Assert.True(await store.ContainsAsync(context, hash1, null));
});
}
}
}
protected async Task <IReadOnlyList <KeyValuePair <ContentHash, ContentFileInfo> > > PopulateRandomInfo(
IContentDirectory contentDirectory, Func <int, bool> shouldPopulateIteration = null)
{
var list = new List <KeyValuePair <ContentHash, ContentFileInfo> >();
for (var i = 0; i < DefaultInfoCount; i++)
{
byte[] content = ThreadSafeRandom.GetBytes(DefaultFileSize);
using (var stream = new MemoryStream(content))
{
ContentHash contentHash = await _hasher.GetContentHashAsync(stream);
var newInfo = new ContentFileInfo(MemoryClock, content.Length);
if (shouldPopulateIteration == null || shouldPopulateIteration(i))
{
await contentDirectory.UpdateAsync(contentHash, false, MemoryClock, fileInfo =>
{
Assert.Null(fileInfo);
return(Task.FromResult(newInfo));
});
}
list.Add(new KeyValuePair <ContentHash, ContentFileInfo>(contentHash, newInfo));
}
}
return(list);
}
public void CreateFromBytes()
{
const int count = 2000000;
var hashType = _hasher.Info.HashType;
var hashBytes = ThreadSafeRandom.GetBytes(_hasher.Info.ByteLength);
PerfTest(); // Warm-up invocation.
var stopwatch = Stopwatch.StartNew();
for (var i = 0; i < count; i++)
{
PerfTest();
}
stopwatch.Stop();
var rate = (long)(count / stopwatch.Elapsed.TotalSeconds);
var name = GetType().Name + "." + nameof(CreateFromBytes);
_resultsFixture.AddResults(Output, name, rate, "items/sec", count);
void PerfTest()
{
var hash = new ContentHash(hashType, hashBytes);
DoSomethingWith(hash);
}
}
public async Task PinUpdatesExpiryWhenFoundRemote()
{
await RunTestAsync(new Context(Logger), 2, async context =>
{
var sessions = context.Sessions;
var session = context.GetDistributedSession(0);
var redisStore = context.GetRedisStore(session);
// Insert random file in session 0 with expiry of 30 minutes
redisStore.ContentHashBumpTime = TimeSpan.FromMinutes(30);
var randomBytes1 = ThreadSafeRandom.GetBytes(0x40);
var putResult1 = await sessions[0].PutStreamAsync(context, HashType.Vso0, new MemoryStream(randomBytes1), Token);
Assert.True(putResult1.Succeeded);
// Insert same file in session 1 and update expiry to 1 hour
redisStore.ContentHashBumpTime = TimeSpan.FromHours(1);
IEnumerable <Task <Indexed <PinResult> > > pinResultTasks = await sessions[1].PinAsync(context, new[] { putResult1.ContentHash }, Token);
Assert.True((await pinResultTasks.First()).Item.Succeeded);
var expiry = await redisStore.GetContentHashExpiryAsync(new Context(Logger), putResult1.ContentHash, CancellationToken.None);
Assert.True(expiry.HasValue);
Assert.InRange(expiry.Value, redisStore.ContentHashBumpTime - TimeSpan.FromSeconds(15), redisStore.ContentHashBumpTime + TimeSpan.FromSeconds(15));
});
}
private void Run(string method, int contentSize, int iterations = Iterations)
{
var content = ThreadSafeRandom.GetBytes(contentSize);
PerfTestCase(); // warm up
var stopwatch = Stopwatch.StartNew();
for (var i = 0; i < iterations; i++)
{
PerfTestCase();
}
stopwatch.Stop();
var rate = (long)(iterations / stopwatch.Elapsed.TotalSeconds);
var name = GetType().Name + "." + method;
_resultsFixture.AddResults(Output, name, rate, "items/sec", iterations);
void PerfTestCase()
{
var x = _hasher.GetContentHash(content);
if (x.ByteLength != _hasher.Info.ByteLength)
{
throw new InvalidOperationException("unexpected hash length");
}
}
}
/// <summary>
/// Put a randomly-sized content from a file into the store.
/// </summary>
public static Task <PutResult> PutRandomFileAsync(
this IContentSession session,
Context context,
IAbsFileSystem fileSystem,
AbsolutePath path,
HashType hashType,
bool provideHash,
long size,
CancellationToken ct)
{
Contract.RequiresNotNull(session);
Contract.RequiresNotNull(context);
Contract.RequiresNotNull(fileSystem);
var c = context.CreateNested(nameof(ContentSessionExtensions));
// TODO: Fix this to work with size > int.Max (bug 1365340)
var data = ThreadSafeRandom.GetBytes((int)size);
fileSystem.WriteAllBytes(path, data);
if (!provideHash)
{
return(session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct));
}
var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);
return(session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct));
}
public Task PutFileExceedQuotaSucceeds()
{
var context = new Context(Logger);
return(TestStore(context, Clock, async store =>
{
int size = MaxSizeHard + ValueSize;
byte[] bytes = ThreadSafeRandom.GetBytes(size);
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
ContentHash hash;
using (var pinContext = store.CreatePinContext())
{
var r = await store.PutFileAsync(
context, pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext)).ShouldBeSuccess();
hash = r.ContentHash;
Clock.Increment();
}
Assert.True(await store.ContainsAsync(context, hash, null));
// Sync to allow calibration to occur.
await store.SyncAsync(context);
var currentQuota = await LoadElasticQuotaAsync(store.RootPathForTest);
Assert.NotNull(currentQuota);
// Calibration should adjust the quota.
Assert.True(size < currentQuota.Quota.Hard);
}
}));
}
public Task RepeatedBlobIsSkipped()
{
ConfigureWithOneMasterAndSmallBlobs();
return(RunTestAsync(
new Context(Logger),
2,
async context =>
{
var sessions = context.Sessions;
var session0 = context.GetDistributedSession(0);
var redisStore0 = context.GetRedisGlobalStore(0);
var session1 = context.GetDistributedSession(1);
var redisStore1 = context.GetRedisGlobalStore(1);
var file = ThreadSafeRandom.GetBytes(10);
var fileString = Encoding.Default.GetString(file);
await session0.PutContentAsync(context, fileString).ShouldBeSuccess();
Assert.Equal(1, redisStore0.Counters[GlobalStoreCounters.PutBlob].Value);
await session1.PutContentAsync(context, fileString).ShouldBeSuccess();
Assert.Equal(1, redisStore1.Counters[GlobalStoreCounters.PutBlob].Value);
Assert.Equal(1, redisStore1.BlobAdapter.Counters[RedisBlobAdapter.RedisBlobAdapterCounters.SkippedBlobs].Value);
}));
}
public async Task PushIsRejectedForTheSameHash()
{
await Task.Yield();
int numberOfPushes = 100;
await RunTestCase(async (rootPath, session, client) =>
{
var bytes = ThreadSafeRandom.GetBytes(1 + 42);
var input = Enumerable.Range(1, numberOfPushes)
.Select(data => (stream: new MemoryStream(bytes), hash: HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(bytes)))
.ToList();
var pushTasks = input.Select(
tpl =>
client.PushFileAsync(
new OperationContext(_context),
tpl.hash,
tpl.stream,
new CopyOptions(bandwidthConfiguration: null))).ToList();
var results = await Task.WhenAll(pushTasks);
results.Any(r => r.Status == CopyResultCode.Rejected_OngoingCopy).Should().BeTrue();
var result = await client.PushFileAsync(
new OperationContext(_context),
input[0].hash,
input[0].stream,
new CopyOptions(bandwidthConfiguration: null));
result.Status.Should().Be(CopyResultCode.Rejected_ContentAvailableLocally);
});
}
public static async Task <PutResult> PutRandomAsync(
this IContentSession session,
Context context,
HashType hashType,
bool provideHash,
long size,
CancellationToken ct)
{
Contract.RequiresNotNull(session);
Contract.RequiresNotNull(context);
var c = context.CreateNested();
// TODO: Fix this to work with size > int.Max (bug 1365340)
var data = ThreadSafeRandom.GetBytes((int)size);
using (var stream = new MemoryStream(data))
{
if (!provideHash)
{
return(await session.PutStreamAsync(c, hashType, stream, ct).ConfigureAwait(false));
}
var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);
return(await session.PutStreamAsync(c, hash, stream, ct).ConfigureAwait(false));
}
}
public Task PutFileHardLinkOverwritesSourceOnMismatchedExistingContentHash()
{
var context = new Context(Logger);
return(TestStore(context, Clock, async store =>
{
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
var bytes = ThreadSafeRandom.GetBytes(100);
var contentHash = bytes.CalculateHash(ContentHashType);
AbsolutePath sourcePath = tempDirectory.CreateRandomFileName();
FileSystem.WriteAllBytes(sourcePath, bytes);
await store.PutFileAsync(context, sourcePath, FileRealizationMode.HardLink, ContentHashType, null).ShouldBeSuccess();
var bytes2 = ThreadSafeRandom.GetBytes(200);
AbsolutePath sourcePath2 = tempDirectory.CreateRandomFileName();
FileSystem.WriteAllBytes(sourcePath2, bytes2);
var result = await store.PutFileAsync(context, sourcePath2, FileRealizationMode.HardLink, contentHash, null);
result.ContentHash.Should().Be(contentHash);
using (StreamWithLength? stream = await FileSystem.OpenAsync(
sourcePath2, FileAccess.Read, FileMode.Open, FileShare.Read))
{
(await stream.Value.CalculateHashAsync(ContentHashType)).Should().Be(contentHash);
}
}
}));
}
public static async Task <PutResult> PutRandomFileAsync(
this IContentSession session,
Context context,
IAbsFileSystem fileSystem,
HashType hashType,
bool provideHash,
long size,
CancellationToken ct)
{
Contract.RequiresNotNull(session);
Contract.RequiresNotNull(context);
Contract.RequiresNotNull(fileSystem);
using (var directory = new DisposableDirectory(fileSystem))
{
var c = context.CreateNested();
// TODO: Fix this to work with size > int.Max (bug 1365340)
var data = ThreadSafeRandom.GetBytes((int)size);
var path = directory.CreateRandomFileName();
fileSystem.WriteAllBytes(path, data);
if (!provideHash)
{
return(await session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct).ConfigureAwait(false));
}
var hash = HashInfoLookup.Find(hashType).CreateContentHasher().GetContentHash(data);
return(await session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct).ConfigureAwait(false));
}
}
public Task PutFilePins()
{
var context = new Context(Logger);
return(TestStore(context, Clock, async store =>
{
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
AbsolutePath pathToContent = tempDirectory.Path / "tempContent.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
ContentHash hashFromPut;
using (var pinContext = store.CreatePinContext())
{
// Put the content into the store w/ hard link
var r = await store.PutFileAsync(
context, pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext));
hashFromPut = r.ContentHash;
Clock.Increment();
await store.EnsureContentIsPinned(context, Clock, hashFromPut);
Assert.True(pinContext.Contains(hashFromPut));
}
await store.EnsureContentIsNotPinned(context, Clock, hashFromPut);
}
}));
}
public Task SuccessivePutStreamsFullPinnedCacheSucceed()
{
return(TestStore(Context, Clock, async store =>
{
using (var pinContext = store.CreatePinContext())
{
var pinRequest = new PinRequest(pinContext);
var r = await store.PutRandomAsync(Context, MaxSizeHard / 3).ShouldBeSuccess();
Clock.Increment();
Assert.True(await store.ContainsAsync(Context, r.ContentHash, pinRequest));
Clock.Increment();
r = await store.PutRandomAsync(Context, MaxSizeHard / 3);
Clock.Increment();
Assert.True(await store.ContainsAsync(Context, r.ContentHash, pinRequest));
Clock.Increment();
var data = ThreadSafeRandom.GetBytes(MaxSizeHard / 2);
using (var dataStream = new MemoryStream(data))
{
await store.PutStreamAsync(Context, dataStream, ContentHashType, null).ShouldBeSuccess();
}
await store.SyncAsync(Context);
}
}));
}
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
FilesCopied.AddOrUpdate(destinationPath, p => path, (dest, prevPath) => overwrite ? path : prevPath);
if (!File.Exists(path.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {path} doesn't exist."));
}
if (File.Exists(destinationPath.Path))
{
if (!overwrite)
{
return(new CopyFileResult(
CopyFileResult.ResultCode.DestinationPathError,
$"Destination file {destinationPath} exists but overwrite not specified."));
}
}
if (FilesToCorrupt.ContainsKey(path))
{
TestGlobal.Logger.Debug($"Corrupting file {path}");
#pragma warning disable AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes
await Task.Run(
() => File.WriteAllBytes(destinationPath.Path, ThreadSafeRandom.GetBytes(150)));
#pragma warning restore AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes
}
else
{
await Task.Run(() => File.Copy(path.Path, destinationPath.Path), cancellationToken);
}
return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length));
}
public Task RepeatedBlobIsSkipped()
{
ConfigureWithOneMasterAndSmallBlobs();
return(RunTestAsync(
new Context(Logger),
2,
async context =>
{
var sessions = context.Sessions;
var session0 = context.GetSession(0);
var redisStore0 = context.GetRedisGlobalStore(0);
var session1 = context.GetSession(1);
var redisStore1 = context.GetRedisGlobalStore(1);
var file = ThreadSafeRandom.GetBytes(10);
var fileString = Encoding.Default.GetString(file);
await session0.PutContentAsync(context, fileString).ShouldBeSuccess();
Assert.Equal(1, redisStore0.Counters[GlobalStoreCounters.PutBlob].Value);
var result = await session1.PutContentAsync(context, fileString).ShouldBeSuccess();
Assert.Equal(1, redisStore1.Counters[GlobalStoreCounters.PutBlob].Value);
var counters = redisStore1.GetBlobAdapter(result.ContentHash).GetCounters().ToDictionaryIntegral();
Assert.Equal(1, counters["SkippedBlobs.Count"]);
}));
}
public async Task EvictionAnnouncesHash()
{
bool batchProcessWasCalled = false;
var nagleQueue = NagleQueue <ContentHash> .Create(
hashes => { batchProcessWasCalled = true; return(Task.FromResult(42)); },
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : 1);
await TestStore(
_context,
_clock,
async store =>
{
var cas = store as IContentStoreInternal;
var blobSize = BlobSizeToStartSoftPurging(2);
using (var stream1 = new MemoryStream(ThreadSafeRandom.GetBytes(blobSize)))
using (var stream2 = new MemoryStream(ThreadSafeRandom.GetBytes(blobSize)))
{
await cas.PutStreamAsync(_context, stream1, ContentHashType).ShouldBeSuccess();
_clock.Increment();
await cas.PutStreamAsync(_context, stream2, ContentHashType).ShouldBeSuccess();
_clock.Increment();
await store.SyncAsync(_context);
}
},
nagleQueue);
batchProcessWasCalled.Should().BeTrue();
}
[Trait("Category", "QTestSkip")] // Skipped
public async Task DeleteOneLinkWhileOneOtherLinkIsOpenReadOnlySharingFailsWithAppropriateError()
{
using (var testDirectory = new DisposableDirectory(FileSystem))
{
var sourcePath = testDirectory.Path / @"source.txt";
var destinationPath1 = testDirectory.Path / @"destination1.txt";
var destinationPath2 = testDirectory.Path / @"destination2.txt";
FileSystem.WriteAllBytes(sourcePath, ThreadSafeRandom.GetBytes(10));
Assert.Equal(CreateHardLinkResult.Success, FileSystem.CreateHardLink(sourcePath, destinationPath1, true));
Assert.Equal(CreateHardLinkResult.Success, FileSystem.CreateHardLink(sourcePath, destinationPath2, true));
using (await FileSystem.OpenAsync(sourcePath, FileAccess.Read, FileMode.Open, ShareReadDelete))
using (await FileSystem.OpenAsync(destinationPath2, FileAccess.Read, FileMode.Open, ShareRead))
{
Action a = () => FileSystem.DeleteFile(destinationPath1);
var exception = Record.Exception(a);
Assert.NotNull(exception);
Assert.IsType <UnauthorizedAccessException>(exception);
Assert.Contains(
"The process cannot access the file because it is being used by another process.",
exception.Message,
StringComparison.OrdinalIgnoreCase);
}
}
}