public Task PinPassThrough() { var context = new Context(Logger); var contentHash = ContentHash.Random(); return(RunMockSessionTestAsync(context, session => { session.PinAsync(context, contentHash, Token, NonDefaultUrgencyHint).ConfigureAwait(false); _mockContentSession.Pinned.Contains(contentHash); return Task.FromResult(0); })); }
public Task OpenStreamPassThrough() { var context = new Context(Logger); var contentHash = ContentHash.Random(); return(RunMockSessionTestAsync(context, session => { session.OpenStreamAsync(context, contentHash, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess(); _mockContentSession.OpenStreamed.Contains(contentHash); return Task.FromResult(0); })); }
public Task PutFileHashPassThrough() { var context = new Context(Logger); var contentHash = ContentHash.Random(); return(RunMockSessionTestAsync(context, session => { session.PutFileAsync(context, contentHash, Path, RealizationMode, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess(); Assert.True(_mockContentSession.PutFileHashParams.Contains(new Tuple <ContentHash, AbsolutePath, FileRealizationMode>(contentHash, Path, RealizationMode)), $"Expected to find ({contentHash},{Path},{RealizationMode}) in set of put files."); return Task.FromResult(0); })); }
public void TestContentEviction() { var clock = new MemoryClock(); var entries = new List <ContentLocationEntry>(); entries.Add( ContentLocationEntry.Create( locations: CreateWithLocationCount(1), // Should be important contentSize: 42, lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2), creationTimeUtc: null)); entries.Add( ContentLocationEntry.Create( locations: CreateWithLocationCount(100), // Maybe important with 3% chance contentSize: 42, lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2), creationTimeUtc: null)); entries.Add( ContentLocationEntry.Create( locations: CreateWithLocationCount(100), // Maybe important with 3% chance contentSize: 42, lastAccessTimeUtc: clock.UtcNow - TimeSpan.FromHours(2), creationTimeUtc: null)); var mock = new EffectiveLastAccessTimeProviderMock(localMachineId: new MachineId(1024)); var hashes = new[] { ContentHash.Random(), ContentHash.Random(), ContentHash.Random() }; mock.Map = new Dictionary <ContentHash, ContentLocationEntry>() { [hashes[0]] = entries[0], [hashes[1]] = entries[1], [hashes[2]] = entries[2], }; var provider = new EffectiveLastAccessTimeProvider(Configuration, clock, mock); var context = new OperationContext(new Context(Logger)); // A given machine id index is higher then the max number of locations used in this test. // This will prevent the provider to consider non-important locations randomly important var input = hashes.Select(hash => new ContentHashWithLastAccessTime(hash, mock.Map[hash].LastAccessTimeUtc.ToDateTime())).ToList(); var result = provider.GetEffectiveLastAccessTimes(context, input).ShouldBeSuccess(); var output = result.Value.ToList(); output.Sort(ContentEvictionInfo.AgeBucketingPrecedenceComparer.Instance); // We know that the first hash should be the last one, because this is only important hash in the list. output[output.Count - 1].ContentHash.Should().Be(hashes[0]); }
public void RoundtripFullBufferPositiveOffset(HashType hashType) { const int offset = 3; var buffer = new byte[ContentHash.SerializedLength + offset]; var h1 = ContentHash.Random(hashType); h1.Serialize(buffer, offset); var h2 = new ContentHash(buffer, offset); Assert.Equal(hashType, h2.HashType); Assert.Equal(h1.ToString(), h2.ToString()); }
public async Task PinNonExisting() { var hashes = Enumerable.Range(0, _itemCount).Select(x => ContentHash.Random()).ToList(); var results = new List <PinResult>(_itemCount); await RunReadOnly( nameof(PinNonExisting), EmptySetupFuncAsync, session => PinAsync(session, hashes, results)); results.All(r => r.Code == PinResult.ResultCode.ContentNotFound).Should().BeTrue(); }
public void SubtractFromItselfShouldNotChangeOrigin() { var hash1 = ContentHash.Random(); var hash2 = ContentHash.Random(); var locations1 = FromHashAndLocations(hash1, new AbsolutePath(Path1)); var locations2 = FromHashAndLocations(hash2, new AbsolutePath(Path2)); var result1 = ToResult(GetBulkOrigin.Local, locations1, locations2); var result2 = result1.Subtract(result1); Assert.Equal(result1.Origin, result2.Origin); }
public void RoundtripPartialBufferPositiveOffset(HashType hashType) { const int offset = 5; var buffer = new byte[ContentHash.SerializedLength + offset]; var h1 = ContentHash.Random(hashType); h1.SerializeHashBytes(buffer, offset); var h2 = new ContentHash(hashType, buffer, offset); Assert.Equal(hashType, h2.HashType); Assert.Equal(h1.ToHex(), h2.ToHex()); }
public void RegisterContentLocationsRequestRoundtrip() { var model = MetadataServiceSerializer.TypeModel; var schema = MetadataServiceSerializer.TypeModel.GetSchema(typeof(RegisterContentLocationsRequest)); var obj = new RegisterContentLocationsRequest() { ContextId = "1", Hashes = new List <ShortHashWithSize>() { (ContentHash.Random(), 42), },
private static ContentHash[] CreateRandomContentHashArray() { Random r = new Random(); var length = r.Next(0, 10); var result = new ContentHash[length]; for (int i = 0; i < length; i++) { result[i] = ContentHash.Random(); } return(result); }
public void EqualContentHashRoundTripViaHexString(HashType hashType) { var h1 = ContentHash.Random(hashType); var hex = h1.ToHex(); var hashInfo = HashInfoLookup.Find(hashType); var buffer = new byte[hashInfo.ByteLength]; var sb = HexUtilities.HexToBytes(hex, buffer); var h2 = new ContentHash(hashType, sb); Assert.Equal(h1, h2); }
public void RandomValue(HashType hashType) { var v = ContentHash.Random(hashType); Assert.Equal(hashType, v.HashType); var hashInfo = HashInfoLookup.Find(hashType); if (hashInfo is TaggedHashInfo taggedHashInfo) { Assert.Equal(v[hashInfo.ByteLength - 1], taggedHashInfo.AlgorithmId); } }
public Task PlaceMiss() { var context = new Context(Logger); return(TestStore(context, async store => { ContentHash contentHash = ContentHash.Random(); AbsolutePath tempPath = _tempDirectory.CreateRandomFileName(); var result = await store.PlaceFileAsync( context, contentHash, tempPath, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.Any); result.Code.Should().Be(PlaceFileResult.ResultCode.NotPlacedContentNotFound); })); }
public void TestSerializeDeserializeSelectorNoOutput() { var selector = new Selector(ContentHash.Random()); var serializerSettings = new JsonSerializerSettings(); serializerSettings.Converters.Add(new SelectorConverter()); string serializedSelector = JsonConvert.SerializeObject(selector, serializerSettings); Selector deserializedSelector = JsonConvert.DeserializeObject <Selector>(serializedSelector, serializerSettings); selector.Should().Be(deserializedSelector); }
public void TestRendererWithHashLookup() { var pathTable = new PathTable(); var fakeContentInfo = FileContentInfo.CreateWithUnknownLength(ContentHash.Random()); var expectedHash = fakeContentInfo.Render(); var renderer = new PipFragmentRenderer(pathTable, (mId) => "XYZ:" + mId, (f) => fakeContentInfo); DoTestRenderer(pathTable, renderer, expectedHash); var moniker = new StringMoniker("123"); XAssert.AreEqual("XYZ:123", renderer.Render(PipFragment.CreateIpcMonikerForTesting(moniker, pathTable.StringTable))); }
public Task PutStreamHashPassThrough() { var context = new Context(Logger); var contentHash = ContentHash.Random(); var stream = new MemoryStream(); return(RunMockSessionTestAsync(context, session => { session.PutStreamAsync(context, contentHash, stream, Token, NonDefaultUrgencyHint).ConfigureAwait(false).GetAwaiter().GetResult().ShouldBeSuccess(); Assert.True(_mockContentSession.PutStreamHashParams.Contains(contentHash), $"Expected to find ({contentHash}) in set of put streams."); return Task.FromResult(0); })); }
public void RareContentShouldBeImportant() { var hash = ContentHash.Random(); var clock = new MemoryClock(); var entry = ContentLocationEntry.Create( locations: new ArrayMachineIdSet(new ushort[1]), contentSize: 42, lastAccessTimeUtc: clock.UtcNow, creationTimeUtc: clock.UtcNow); var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(1), Configuration, clock.UtcNow); rank.Should().Be(ReplicaRank.Important); }
public void RareContentShouldBeImportant() { var hash = ContentHash.Random(); var clock = new MemoryClock(); var entry = ContentLocationEntry.Create( locations: new ArrayMachineIdSet(new ushort[1]), contentSize: 42, lastAccessTimeUtc: clock.UtcNow, creationTimeUtc: clock.UtcNow); bool isImportant = EffectiveLastAccessTimeProvider.IsImportantReplica(hash, entry, new MachineId(1), Configuration.DesiredReplicaRetention); isImportant.Should().BeTrue(); }
private IEnumerable <FileInfo> EnumerateFiles(AbsolutePath root) { for (var i = 0; i < OOMEnumerationFiles; i++) { var hash = ContentHash.Random().ToHex(); string text = i.ToString(); yield return(new FileInfo { FullPath = root / "VSO0" / $"{hash.Substring(0, 3)}" / $"{hash}.blob", Length = 1, }); } }
public void Success() { var contentSize = 9; var contentHash = ContentHash.Random(); var deleteResult = new DeleteResult(DeleteResult.ResultCode.Success, contentHash, contentSize); var deleteResultMapping = new Dictionary <string, DeleteResult> { { "TEST_MACHINE_LOCATION", deleteResult } }; var distributedDeleteResult = new DistributedDeleteResult(contentHash, contentSize, deleteResultMapping); distributedDeleteResult.ToString().Should().Contain("ContentSize"); distributedDeleteResult.ToString().Should().Contain("Success"); }
public void TestToString() { // Hash.ToString is a very important method, because the result of it is used as keys in Redis. var hash = ContentHash.Random(HashType.Vso0); var shortHash = new ShortHash(hash); hash.ToString().Should().Contain(shortHash.ToString()); var sb = new StringBuilder(); shortHash.ToString(sb); shortHash.ToString().Should().BeEquivalentTo(sb.ToString()); }
public void Error() { var contentSize = 0; var contentHash = ContentHash.Random(); var deleteResult = new DeleteResult(DeleteResult.ResultCode.Error, "errorMsg", "reason"); var deleteResultMapping = new Dictionary <string, DeleteResult> { { "TEST_MACHINE_LOCATION", deleteResult } }; var distributedDeleteResult = new DistributedDeleteResult(contentHash, contentSize, deleteResultMapping); distributedDeleteResult.ToString().Should().Contain("errorMsg"); distributedDeleteResult.ToString().Should().Contain("reason"); }
public void ContentNotFound() { var contentSize = 0; var contentHash = ContentHash.Random(); var deleteResult = new DeleteResult(DeleteResult.ResultCode.ContentNotFound, contentHash, contentSize); var deleteResultMapping = new Dictionary <string, DeleteResult>(); deleteResultMapping.Add("TEST_MACHINE_LOCATION", deleteResult); var distributedDeleteResult = new DistributedDeleteResult(contentHash, contentSize, deleteResultMapping); distributedDeleteResult.ToString().Should().Contain("size could not be determined"); distributedDeleteResult.ToString().Should().Contain("ContentNotFound"); }
public async Task WrongPort() { await RunTestCase(nameof(WrongPort), async (rootPath, session, client) => { // Copy fake file out via GRPC var host = "localhost"; var bogusPort = PortExtensions.GetNextAvailablePort(); using (client = GrpcCopyClient.Create(host, bogusPort)) { var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None); Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code); } }); }
public void TestVolatileWithShortHashSet() { var clock = new MemoryClock(); var set = new VolatileSet <ShortHash>(clock); for (int i = 0; i < 500_000; i++) { var hash = ContentHash.Random(HashType.Vso0); var shortHash = new ShortHash(hash); set.Add(shortHash, TimeSpan.FromMinutes(30)); set.Contains(shortHash).Should().BeTrue(); } }
public void RareContentShouldBeImportantOrProtectedWithThrottling(ushort[] machineIds) { Configuration.ThrottledEvictionInterval = TimeSpan.FromMinutes(20); var hash = ContentHash.Random(); var clock = new MemoryClock(); var entry = ContentLocationEntry.Create( locations: new ArrayMachineIdSet(machineIds), contentSize: 42, lastAccessTimeUtc: clock.UtcNow, creationTimeUtc: clock.UtcNow); int totalImportantFound = 0; foreach (var machineId in machineIds) { int protectedCount = 0; int importantCount = 0; for (int i = 0; i < Configuration.DesiredReplicaRetention; i++) { var rank = EffectiveLastAccessTimeProvider.GetReplicaRank(hash, entry, new MachineId(machineId), Configuration, clock.UtcNow); clock.UtcNow += Configuration.ThrottledEvictionInterval; switch (rank) { case ReplicaRank.Important: importantCount++; totalImportantFound++; break; case ReplicaRank.Protected: protectedCount++; break; default: XAssert.Fail($"Rank is '{rank}' but should be Important or Protected since content is rare"); break; } } protectedCount.Should().BeInRange(Configuration.DesiredReplicaRetention - 1, Configuration.DesiredReplicaRetention, "At least (DesiredReplicaCount - 1) out of DesiredReplicaCount of the time content should be protected"); importantCount.Should().BeInRange(0, 1, "At most 1 out of DesiredReplicaCount of the time content should just be important allowing eviction"); } totalImportantFound.Should().Be(1, "Within the time interval (DesiredReplicaCount * ThrottledEvictionInterval), " + "during exactly one ThrottledEvictionInterval exactly one machine should have content unproteced"); }
public Task WrongPort() { return(RunTestCase(async(rootPath, session, client) => { // Copy fake file out via GRPC var bogusPort = PortExtensions.GetNextAvailablePort(); await _clientCache.UseAsync(new OperationContext(_context), LocalHost, bogusPort, async(nestedContext, client) => { var copyFileResult = await client.CopyFileAsync(nestedContext, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), new CopyOptions(bandwidthConfiguration: null)); Assert.Equal(CopyResultCode.ServerUnavailable, copyFileResult.Code); return Unit.Void; }); })); }
public Task PutFileWithMissingFileReturnsError() { var context = new Context(Logger); return TestStore(context, Clock, async store => { var contentHash = ContentHash.Random(); using (var tempDirectory = new DisposableDirectory(FileSystem)) { var r = await store.PutFileAsync( context, tempDirectory.CreateRandomFileName(), FileRealizationMode.Any, contentHash, null); r.Succeeded.Should().BeFalse(); r.ErrorMessage.Should().Contain("Source file not found"); } }); }
public void ContentLazilyUpdated() { Context context = new Context(m_logger); TimeSpan contentHashBumpTime = TimeSpan.FromHours(1); ContentTrackerUpdater contentTrackerUpdater = new ContentTrackerUpdater(hashes => { }, contentHashBumpTime); ContentHash hash = ContentHash.Random(); long size = 0; DateTime lastAccessTime = DateTime.MaxValue; // Content hash will always be above the age limit. ContentHashWithSizeAndLastAccessTime contentHashInfo = new ContentHashWithSizeAndLastAccessTime(hash, size, lastAccessTime); var eagerUpdate = contentTrackerUpdater.ScheduleHashTouches(context, new[] { contentHashInfo }); Assert.Empty(eagerUpdate); }
public void TestContentHashListsNoPayloadOrderInvalidation() { var vsoContentHash1 = ContentHash.Random(); var vsoContentHash2 = ContentHash.Random(); var contentHashList = new ContentHashList(new[] { vsoContentHash1, vsoContentHash2 }); byte[] hashOfContentHashes = contentHashList.GetHashOfHashes(); var secondOrderContentHashList = new ContentHashList(new[] { vsoContentHash2, vsoContentHash1 }); byte[] secondOrderHashOfContentHashes = secondOrderContentHashList.GetHashOfHashes(); ByteArrayComparer.ArraysEqual(hashOfContentHashes, secondOrderHashOfContentHashes).Should().BeFalse(); }