public Task PinPassThrough()
{
var context = new Context(Logger);
var contentHash = ContentHash.Random();
return(RunMockSessionTestAsync(context, session =>
{
session.PinAsync(context, contentHash, Token, NonDefaultUrgencyHint).ConfigureAwait(false);
_mockContentSession.Pinned.Contains(contentHash);
return Task.FromResult(0);
}));
}
public Task OpenStreamPassThrough()
{
var context = new Context(Logger);
var contentHash = ContentHash.Random();
return(RunMockSessionTestAsync(context, session =>
{
session.OpenStreamAsync(context, contentHash, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess();
_mockContentSession.OpenStreamed.Contains(contentHash);
return Task.FromResult(0);
}));
}
public Task PutFileHashPassThrough()
{
var context = new Context(Logger);
var contentHash = ContentHash.Random();
return(RunMockSessionTestAsync(context, session =>
{
session.PutFileAsync(context, contentHash, Path, RealizationMode, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess();
Assert.True(_mockContentSession.PutFileHashParams.Contains(new Tuple <ContentHash, AbsolutePath, FileRealizationMode>(contentHash, Path, RealizationMode)), $"Expected to find ({contentHash},{Path},{RealizationMode}) in set of put files.");
return Task.FromResult(0);
}));
}
public void TestContentEviction()
{
var clock = new MemoryClock();
var entries = new List <ContentLocationEntry>();
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(1), // Should be important
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100), // Maybe important with 3% chance
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
entries.Add(
ContentLocationEntry.Create(
locations: CreateWithLocationCount(100), // Maybe important with 3% chance
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2),
creationTimeUtc: null));
var mock = new EffectiveLastAccessTimeProviderMock(localMachineId: new MachineId(1024));
var hashes = new[] { ContentHash.Random(), ContentHash.Random(), ContentHash.Random() };
mock.Map = new Dictionary <ContentHash, ContentLocationEntry>()
{
[hashes[0]] = entries[0],
[hashes[1]] = entries[1],
[hashes[2]] = entries[2],
};
var provider = new EffectiveLastAccessTimeProvider(Configuration, clock, mock);
var context = new OperationContext(new Context(Logger));
// A given machine id index is higher then the max number of locations used in this test.
// This will prevent the provider to consider non-important locations randomly important
var input = hashes.Select(hash => new ContentHashWithLastAccessTime(hash, mock.Map[hash].LastAccessTimeUtc.ToDateTime())).ToList();
var result = provider.GetEffectiveLastAccessTimes(context, input).ShouldBeSuccess();
var output = result.Value.ToList();
output.Sort(ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance);
// We know that the first hash should be the last one, because this is only important hash in the list.
output[output.Count - 1].ContentHash.Should().Be(hashes[0]);
}
public void RoundtripFullBufferPositiveOffset(HashType hashType)
{
const int offset = 3;
var buffer = new byte[ContentHash.SerializedLength + offset];
var h1 = ContentHash.Random(hashType);
h1.Serialize(buffer, offset);
var h2 = new ContentHash(buffer, offset);
Assert.Equal(hashType, h2.HashType);
Assert.Equal(h1.ToString(), h2.ToString());
}
public async Task PinNonExisting()
{
var hashes = Enumerable.Range(0, _itemCount).Select(x => ContentHash.Random()).ToList();
var results = new List <PinResult>(_itemCount);
await RunReadOnly(
nameof(PinNonExisting),
EmptySetupFuncAsync,
session => PinAsync(session, hashes, results));
results.All(r => r.Code == PinResult.ResultCode.ContentNotFound).Should().BeTrue();
}
public void SubtractFromItselfShouldNotChangeOrigin()
{
var hash1 = ContentHash.Random();
var hash2 = ContentHash.Random();
var locations1 = FromHashAndLocations(hash1, new AbsolutePath(Path1));
var locations2 = FromHashAndLocations(hash2, new AbsolutePath(Path2));
var result1 = ToResult(GetBulkOrigin.Local, locations1, locations2);
var result2 = result1.Subtract(result1);
Assert.Equal(result1.Origin, result2.Origin);
}
public void RoundtripPartialBufferPositiveOffset(HashType hashType)
{
const int offset = 5;
var buffer = new byte[ContentHash.SerializedLength + offset];
var h1 = ContentHash.Random(hashType);
h1.SerializeHashBytes(buffer, offset);
var h2 = new ContentHash(hashType, buffer, offset);
Assert.Equal(hashType, h2.HashType);
Assert.Equal(h1.ToHex(), h2.ToHex());
}
public void RegisterContentLocationsRequestRoundtrip()
{
var model = MetadataServiceSerializer.TypeModel;
var schema = MetadataServiceSerializer.TypeModel.GetSchema(typeof(RegisterContentLocationsRequest));
var obj = new RegisterContentLocationsRequest()
{
ContextId = "1",
Hashes = new List <ShortHashWithSize>()
{
(ContentHash.Random(), 42),
},
private static ContentHash[] CreateRandomContentHashArray()
{
Random r = new Random();
var length = r.Next(0, 10);
var result = new ContentHash[length];
for (int i = 0; i < length; i++)
{
result[i] = ContentHash.Random();
}
return(result);
}
public void EqualContentHashRoundTripViaHexString(HashType hashType)
{
var h1 = ContentHash.Random(hashType);
var hex = h1.ToHex();
var hashInfo = HashInfoLookup.Find(hashType);
var buffer = new byte[hashInfo.ByteLength];
var sb = HexUtilities.HexToBytes(hex, buffer);
var h2 = new ContentHash(hashType, sb);
Assert.Equal(h1, h2);
}
public void RandomValue(HashType hashType)
{
var v = ContentHash.Random(hashType);
Assert.Equal(hashType, v.HashType);
var hashInfo = HashInfoLookup.Find(hashType);
if (hashInfo is TaggedHashInfo taggedHashInfo)
{
Assert.Equal(v[hashInfo.ByteLength - 1], taggedHashInfo.AlgorithmId);
}
}
public Task PlaceMiss()
{
var context = new Context(Logger);
return(TestStore(context, async store =>
{
ContentHash contentHash = ContentHash.Random();
AbsolutePath tempPath = _tempDirectory.CreateRandomFileName();
var result = await store.PlaceFileAsync(
context, contentHash, tempPath, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.Any);
result.Code.Should().Be(PlaceFileResult.ResultCode.NotPlacedContentNotFound);
}));
}
public void TestSerializeDeserializeSelectorNoOutput()
{
var selector = new Selector(ContentHash.Random());
var serializerSettings = new JsonSerializerSettings();
serializerSettings.Converters.Add(new SelectorConverter());
string serializedSelector = JsonConvert.SerializeObject(selector, serializerSettings);
Selector deserializedSelector =
JsonConvert.DeserializeObject <Selector>(serializedSelector, serializerSettings);
selector.Should().Be(deserializedSelector);
}
public void TestRendererWithHashLookup()
{
var pathTable = new PathTable();
var fakeContentInfo = FileContentInfo.CreateWithUnknownLength(ContentHash.Random());
var expectedHash = fakeContentInfo.Render();
var renderer = new PipFragmentRenderer(pathTable, (mId) => "XYZ:" + mId, (f) => fakeContentInfo);
DoTestRenderer(pathTable, renderer, expectedHash);
var moniker = new StringMoniker("123");
XAssert.AreEqual("XYZ:123", renderer.Render(PipFragment.CreateIpcMonikerForTesting(moniker, pathTable.StringTable)));
}
public Task PutStreamHashPassThrough()
{
var context = new Context(Logger);
var contentHash = ContentHash.Random();
var stream = new MemoryStream();
return(RunMockSessionTestAsync(context, session =>
{
session.PutStreamAsync(context, contentHash, stream, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess();
Assert.True(_mockContentSession.PutStreamHashParams.Contains(contentHash), $"Expected to find ({contentHash}) in set of put streams.");
return Task.FromResult(0);
}));
}
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(1), Configuration, clock.UtcNow);
rank.Should().Be(ReplicaRank.Important);
}
public void RareContentShouldBeImportant()
{
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(new ushort[1]),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
bool isImportant = EffectiveLastAccessTimeProvider.IsImportantReplica(hash, entry, new MachineId(1), Configuration.DesiredReplicaRetention);
isImportant.Should().BeTrue();
}
private IEnumerable <FileInfo> EnumerateFiles(AbsolutePath root)
{
for (var i = 0; i < OOMEnumerationFiles; i++)
{
var hash = ContentHash.Random().ToHex();
string text = i.ToString();
yield return(new FileInfo {
FullPath = root / "VSO0" / $"{hash.Substring(0, 3)}" / $"{hash}.blob",
Length = 1,
});
}
}
public void Success()
{
var contentSize = 9;
var contentHash = ContentHash.Random();
var deleteResult = new DeleteResult(DeleteResult.ResultCode.Success, contentHash, contentSize);
var deleteResultMapping = new Dictionary <string, DeleteResult> {
{ "TEST_MACHINE_LOCATION", deleteResult }
};
var distributedDeleteResult = new DistributedDeleteResult(contentHash, contentSize, deleteResultMapping);
distributedDeleteResult.ToString().Should().Contain("ContentSize");
distributedDeleteResult.ToString().Should().Contain("Success");
}
public void TestToString()
{
// Hash.ToString is a very important method, because the result of it is used as keys in Redis.
var hash = ContentHash.Random(HashType.Vso0);
var shortHash = new ShortHash(hash);
hash.ToString().Should().Contain(shortHash.ToString());
var sb = new StringBuilder();
shortHash.ToString(sb);
shortHash.ToString().Should().BeEquivalentTo(sb.ToString());
}
public void Error()
{
var contentSize = 0;
var contentHash = ContentHash.Random();
var deleteResult = new DeleteResult(DeleteResult.ResultCode.Error, "errorMsg", "reason");
var deleteResultMapping = new Dictionary <string, DeleteResult> {
{ "TEST_MACHINE_LOCATION", deleteResult }
};
var distributedDeleteResult = new DistributedDeleteResult(contentHash, contentSize, deleteResultMapping);
distributedDeleteResult.ToString().Should().Contain("errorMsg");
distributedDeleteResult.ToString().Should().Contain("reason");
}
public void ContentNotFound()
{
var contentSize = 0;
var contentHash = ContentHash.Random();
var deleteResult = new DeleteResult(DeleteResult.ResultCode.ContentNotFound, contentHash, contentSize);
var deleteResultMapping = new Dictionary <string, DeleteResult>();
deleteResultMapping.Add("TEST_MACHINE_LOCATION", deleteResult);
var distributedDeleteResult = new DistributedDeleteResult(contentHash, contentSize, deleteResultMapping);
distributedDeleteResult.ToString().Should().Contain("size could not be determined");
distributedDeleteResult.ToString().Should().Contain("ContentNotFound");
}
public async Task WrongPort()
{
await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
{
// Copy fake file out via GRPC
var host = "localhost";
var bogusPort = PortExtensions.GetNextAvailablePort();
using (client = GrpcCopyClient.Create(host, bogusPort))
{
var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
}
});
}
public void TestVolatileWithShortHashSet()
{
var clock = new MemoryClock();
var set = new VolatileSet <ShortHash>(clock);
for (int i = 0; i < 500_000; i++)
{
var hash = ContentHash.Random(HashType.Vso0);
var shortHash = new ShortHash(hash);
set.Add(shortHash, TimeSpan.FromMinutes(30));
set.Contains(shortHash).Should().BeTrue();
}
}
public void RareContentShouldBeImportantOrProtectedWithThrottling(ushort[] machineIds)
{
Configuration.ThrottledEvictionInterval = TimeSpan.FromMinutes(20);
var hash = ContentHash.Random();
var clock = new MemoryClock();
var entry = ContentLocationEntry.Create(
locations: new ArrayMachineIdSet(machineIds),
contentSize: 42,
lastAccessTimeUtc: clock.UtcNow,
creationTimeUtc: clock.UtcNow);
int totalImportantFound = 0;
foreach (var machineId in machineIds)
{
int protectedCount = 0;
int importantCount = 0;
for (int i = 0; i < Configuration.DesiredReplicaRetention; i++)
{
var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(machineId), Configuration, clock.UtcNow);
clock.UtcNow += Configuration.ThrottledEvictionInterval;
switch (rank)
{
case ReplicaRank.Important:
importantCount++;
totalImportantFound++;
break;
case ReplicaRank.Protected:
protectedCount++;
break;
default:
XAssert.Fail($"Rank is '{rank}' but should be Important or Protected since content is rare");
break;
}
}
protectedCount.Should().BeInRange(Configuration.DesiredReplicaRetention - 1, Configuration.DesiredReplicaRetention, "At least (DesiredReplicaCount - 1) out of DesiredReplicaCount of the time content should be protected");
importantCount.Should().BeInRange(0, 1, "At most 1 out of DesiredReplicaCount of the time content should just be important allowing eviction");
}
totalImportantFound.Should().Be(1, "Within the time interval (DesiredReplicaCount * ThrottledEvictionInterval), " +
"during exactly one ThrottledEvictionInterval exactly one machine should have content unproteced");
}
public Task WrongPort()
{
return(RunTestCase(async(rootPath, session, client) =>
{
// Copy fake file out via GRPC
var bogusPort = PortExtensions.GetNextAvailablePort();
await _clientCache.UseAsync(new OperationContext(_context), LocalHost, bogusPort, async(nestedContext, client) =>
{
var copyFileResult = await client.CopyFileAsync(nestedContext, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), new CopyOptions(bandwidthConfiguration: null));
Assert.Equal(CopyResultCode.ServerUnavailable, copyFileResult.Code);
return Unit.Void;
});
}));
}
public Task PutFileWithMissingFileReturnsError()
{
var context = new Context(Logger);
return TestStore(context, Clock, async store =>
{
var contentHash = ContentHash.Random();
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
var r = await store.PutFileAsync(
context, tempDirectory.CreateRandomFileName(), FileRealizationMode.Any, contentHash, null);
r.Succeeded.Should().BeFalse();
r.ErrorMessage.Should().Contain("Source file not found");
}
});
}
public void ContentLazilyUpdated()
{
Context context = new Context(m_logger);
TimeSpan contentHashBumpTime = TimeSpan.FromHours(1);
ContentTrackerUpdater contentTrackerUpdater = new ContentTrackerUpdater(hashes => { }, contentHashBumpTime);
ContentHash hash = ContentHash.Random();
long size = 0;
DateTime lastAccessTime = DateTime.MaxValue; // Content hash will always be above the age limit.
ContentHashWithSizeAndLastAccessTime contentHashInfo = new ContentHashWithSizeAndLastAccessTime(hash, size, lastAccessTime);
var eagerUpdate = contentTrackerUpdater.ScheduleHashTouches(context, new[] { contentHashInfo });
Assert.Empty(eagerUpdate);
}
public void TestContentHashListsNoPayloadOrderInvalidation()
{
var vsoContentHash1 = ContentHash.Random();
var vsoContentHash2 = ContentHash.Random();
var contentHashList = new ContentHashList(new[] { vsoContentHash1, vsoContentHash2 });
byte[] hashOfContentHashes = contentHashList.GetHashOfHashes();
var secondOrderContentHashList = new ContentHashList(new[] { vsoContentHash2, vsoContentHash1 });
byte[] secondOrderHashOfContentHashes = secondOrderContentHashList.GetHashOfHashes();
ByteArrayComparer.ArraysEqual(hashOfContentHashes, secondOrderHashOfContentHashes).Should().BeFalse();
}
