public async void FreeSpaceCombination() { File.Delete("test.blob"); using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); var c2 = await s.AddChunk(ChunkTypes.Test, 12, data, CancellationToken.None); await s.AddChunk(ChunkTypes.Test, 13, data, CancellationToken.None); Assert.Equal(3, (await s.GetChunks(CancellationToken.None)).Count); await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c2.Id), CancellationToken.None); var chunks = await s.GetChunks(CancellationToken.None); Assert.Equal(3, chunks.Count); Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free)); await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None); chunks = await s.GetChunks(CancellationToken.None); Assert.Equal(2, chunks.Count); Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free)); } File.Delete("test.blob"); using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); var c2 = await s.AddChunk(ChunkTypes.Test, 12, data, CancellationToken.None); await s.AddChunk(ChunkTypes.Test, 13, data, CancellationToken.None); Assert.Equal(3, (await s.GetChunks(CancellationToken.None)).Count); await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None); var chunks = await s.GetChunks(CancellationToken.None); Assert.Equal(3, chunks.Count); Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free)); await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c2.Id), CancellationToken.None); chunks = await s.GetChunks(CancellationToken.None); Assert.Equal(2, chunks.Count); Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free)); } }
public async void CrcHead() { File.Delete("testcrc.blob"); using (var s = new BlobStorage("testcrc.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); } using (var f = File.OpenWrite("testcrc.blob")) { f.Position = 32; f.WriteByte(5); } using (var s = new BlobStorage("testcrc.blob")) { await Assert.ThrowsAsync <InvalidDataException>(() => s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); s.TruncateOnChunkInitializationError = true; Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); s.Info.Refresh(); Assert.Equal(BlobStorage.HeaderSize, s.Info.Length); } }
public async void Cleanup() { File.Delete("cache.blob"); var storage = new BlobStorage("cache.blob"); await storage.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None); using (var c = new Cache(storage)) { c.CleanupTime = () => DateTime.UtcNow.AddDays(2); Assert.True(await c.Initialize(CancellationToken.None)); await c.Add("xunit.core.xml", DateTime.MaxValue, File.ReadAllBytes("xunit.core.xml"), CancellationToken.None); await c.Add("xunit.assert.xml", DateTime.UtcNow.AddMinutes(2), File.ReadAllBytes("xunit.assert.xml"), CancellationToken.None); await storage.AddChunk(ChunkTypes.Data, 8, new byte[] { 1, 2, 3 }, CancellationToken.None); storage.Info.Refresh(); var length = storage.Info.Length; await c.Cleanup(CancellationToken.None); var chunks = await storage.GetChunks(CancellationToken.None); Assert.Equal(1, chunks.Count(ch => ch.Type == ChunkTypes.Head)); Assert.Equal(1, chunks.Count(ch => ch.Type == ChunkTypes.Data)); storage.Info.Refresh(); Assert.True(length > storage.Info.Length); } }
public async void SpaceReuse() { File.Delete("test.blob"); using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); Assert.Equal(11u, c1.UserData); Assert.Equal((uint)data.Length, c1.Size); var res = await s.ReadChunks(c => c.Id == c1.Id, CancellationToken.None); Assert.Equal(data, res.First().Data); data = Enumerable.Range(0, 256).Select(r => (byte)2).ToArray(); var c2 = await s.AddChunk(ChunkTypes.Test, 12, data, CancellationToken.None); Assert.Equal(12u, c2.UserData); Assert.Equal((uint)data.Length, c2.Size); res = await s.ReadChunks(c => c.Id == c2.Id, CancellationToken.None); Assert.Equal(data, res.First().Data); s.Info.Refresh(); var size = s.Info.Length; await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None); data = Enumerable.Range(0, 128).Select(r => (byte)3).ToArray(); var c3 = await s.AddChunk(ChunkTypes.Test, 13, data, CancellationToken.None); Assert.Equal(13u, c3.UserData); Assert.Equal((uint)data.Length, c3.Size); res = await s.ReadChunks(c => c.Id == c3.Id, CancellationToken.None); Assert.Equal(data, res.First().Data); Assert.Equal(size, s.Info.Length); } }
private static async Task ConcurrencyRemove <T>() where T : ConcurrencyHandler, new() { File.Delete($"concurrencyRemove-{typeof(T).Name}.blob"); var rn = new Random(); using (var s = new BlobStorage($"concurrencyRemove-{typeof(T).Name}.blob")) { await s.Initialize <T>(CancellationToken.None); for (uint j = 0; j < 50; j++) { await s.AddChunk(ChunkTypes.Test, j, Enumerable.Range(0, rn.Next(500) + 500).Select(_ => (byte)rn.Next(256)).ToArray(), CancellationToken.None); } } var counter = 0; const int max = 25; for (var i = 0; i < max; i++) { var t = new Thread(() => { try { var r = new Random(); using (var s = new BlobStorage($"concurrencyRemove-{typeof(T).Name}.blob")) { s.Initialize <T>(CancellationToken.None).Wait(); for (uint j = 0; j < 40; j++) { s.RemoveChunk(si => si.Chunks.Count == 0 ? (StorageChunk?)null : si.Chunks[r.Next(si.Chunks.Count)], CancellationToken.None).Wait(); } } } finally { Interlocked.Increment(ref counter); } }); t.Start(); Thread.Sleep(100); } // ReSharper disable once LoopVariableIsNeverChangedInsideLoop while (counter < max) { Thread.Sleep(100); } using (var s = new BlobStorage($"concurrencyRemove-{typeof(T).Name}.blob")) { Assert.True(await s.Initialize <T>(CancellationToken.None)); } }
public async void StreamRead() { File.Delete("test.blob"); using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); using (var ms = new MemoryStream()) { // ReSharper disable once AccessToDisposedClosure var res = await s.ReadChunks(c => c.Id == c1.Id, c => ms, CancellationToken.None); Assert.Equal(data, ms.ToArray()); Assert.Equal(ms, res.First().Data); } using (var ms = new MemoryStream()) { // ReSharper disable once AccessToDisposedClosure var res = await s.ReadChunks(sc => sc.Chunks.Where(c => c.Id == c1.Id), c => ms, CancellationToken.None); Assert.Equal(data, ms.ToArray()); Assert.Equal(ms, res.First().Data); } using (var ms = new MemoryStream()) { // ReSharper disable once AccessToDisposedClosure var res = await s.ReadChunks(c => c.Id == c1.Id?ms : null, CancellationToken.None); Assert.Equal(data, ms.ToArray()); Assert.Equal(ms, res.First().Data); } using (var ms = new MemoryStream()) { // ReSharper disable once AccessToDisposedClosure var res = await s.ReadChunks(sc => sc.Chunks.Where(c => c.Id == c1.Id).Select(c => (c, (Stream)ms)), CancellationToken.None); Assert.Equal(data, ms.ToArray()); Assert.Equal(ms, res.First().Data); } } }
public async void AddChunk() { File.Delete("test.blob"); using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); Assert.Equal(1u, c1.Id); Assert.Equal(11u, c1.UserData); Assert.Equal((uint)data.Length, c1.Size); var res = await s.ReadChunks(c => c.Id == 1, CancellationToken.None); Assert.Equal(data, res.First().Data); } }
public async void GetChunks() { File.Delete("test.blob"); using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); Assert.Empty(await s.GetChunks(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); Assert.Equal(c1, (await s.GetChunks(CancellationToken.None)).Single()); await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None); Assert.Equal(c1.Id, (await s.GetChunks(CancellationToken.None)).Single().Id); Assert.Equal(ChunkTypes.Free, (await s.GetChunks(CancellationToken.None)).Single().Type); } }
public async void RemoveChunk() { using (var s = new BlobStorage("test.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); Assert.Equal(11u, c1.UserData); Assert.Equal((uint)data.Length, c1.Size); var res = await s.ReadChunks(c => c.Id == c1.Id, CancellationToken.None); Assert.Equal(data, res.First().Data); await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None); var chunks = await s.GetChunks(CancellationToken.None); Assert.Equal(ChunkTypes.Free, chunks.Single(c => c.Id == c1.Id).Type); } }
public async void CrcData() { File.Delete("testcrc.blob"); using (var s = new BlobStorage("testcrc.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray(); await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None); } using (var f = File.OpenWrite("testcrc.blob")) { f.Position = 150; f.WriteByte(5); } using (var s = new BlobStorage("testcrc.blob")) { Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None)); await Assert.ThrowsAsync <InvalidDataException>(() => s.ReadChunks(sc => true, CancellationToken.None)); } }