public async void FreeSpaceCombination()
{
File.Delete("test.blob");
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
var c2 = await s.AddChunk(ChunkTypes.Test, 12, data, CancellationToken.None);
await s.AddChunk(ChunkTypes.Test, 13, data, CancellationToken.None);
Assert.Equal(3, (await s.GetChunks(CancellationToken.None)).Count);
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c2.Id), CancellationToken.None);
var chunks = await s.GetChunks(CancellationToken.None);
Assert.Equal(3, chunks.Count);
Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free));
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None);
chunks = await s.GetChunks(CancellationToken.None);
Assert.Equal(2, chunks.Count);
Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free));
}
File.Delete("test.blob");
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
var c2 = await s.AddChunk(ChunkTypes.Test, 12, data, CancellationToken.None);
await s.AddChunk(ChunkTypes.Test, 13, data, CancellationToken.None);
Assert.Equal(3, (await s.GetChunks(CancellationToken.None)).Count);
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None);
var chunks = await s.GetChunks(CancellationToken.None);
Assert.Equal(3, chunks.Count);
Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free));
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c2.Id), CancellationToken.None);
chunks = await s.GetChunks(CancellationToken.None);
Assert.Equal(2, chunks.Count);
Assert.Equal(1, chunks.Count(c => c.Type == ChunkTypes.Free));
}
}
public async void CrcHead()
{
File.Delete("testcrc.blob");
using (var s = new BlobStorage("testcrc.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
}
using (var f = File.OpenWrite("testcrc.blob"))
{
f.Position = 32;
f.WriteByte(5);
}
using (var s = new BlobStorage("testcrc.blob"))
{
await Assert.ThrowsAsync <InvalidDataException>(() => s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
s.TruncateOnChunkInitializationError = true;
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
s.Info.Refresh();
Assert.Equal(BlobStorage.HeaderSize, s.Info.Length);
}
}
public async void Cleanup()
{
File.Delete("cache.blob");
var storage = new BlobStorage("cache.blob");
await storage.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None);
using (var c = new Cache(storage))
{
c.CleanupTime = () => DateTime.UtcNow.AddDays(2);
Assert.True(await c.Initialize(CancellationToken.None));
await c.Add("xunit.core.xml", DateTime.MaxValue, File.ReadAllBytes("xunit.core.xml"), CancellationToken.None);
await c.Add("xunit.assert.xml", DateTime.UtcNow.AddMinutes(2), File.ReadAllBytes("xunit.assert.xml"), CancellationToken.None);
await storage.AddChunk(ChunkTypes.Data, 8, new byte[] { 1, 2, 3 }, CancellationToken.None);
storage.Info.Refresh();
var length = storage.Info.Length;
await c.Cleanup(CancellationToken.None);
var chunks = await storage.GetChunks(CancellationToken.None);
Assert.Equal(1, chunks.Count(ch => ch.Type == ChunkTypes.Head));
Assert.Equal(1, chunks.Count(ch => ch.Type == ChunkTypes.Data));
storage.Info.Refresh();
Assert.True(length > storage.Info.Length);
}
}
public async void SpaceReuse()
{
File.Delete("test.blob");
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
Assert.Equal(11u, c1.UserData);
Assert.Equal((uint)data.Length, c1.Size);
var res = await s.ReadChunks(c => c.Id == c1.Id, CancellationToken.None);
Assert.Equal(data, res.First().Data);
data = Enumerable.Range(0, 256).Select(r => (byte)2).ToArray();
var c2 = await s.AddChunk(ChunkTypes.Test, 12, data, CancellationToken.None);
Assert.Equal(12u, c2.UserData);
Assert.Equal((uint)data.Length, c2.Size);
res = await s.ReadChunks(c => c.Id == c2.Id, CancellationToken.None);
Assert.Equal(data, res.First().Data);
s.Info.Refresh();
var size = s.Info.Length;
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None);
data = Enumerable.Range(0, 128).Select(r => (byte)3).ToArray();
var c3 = await s.AddChunk(ChunkTypes.Test, 13, data, CancellationToken.None);
Assert.Equal(13u, c3.UserData);
Assert.Equal((uint)data.Length, c3.Size);
res = await s.ReadChunks(c => c.Id == c3.Id, CancellationToken.None);
Assert.Equal(data, res.First().Data);
Assert.Equal(size, s.Info.Length);
}
}
private static async Task ConcurrencyRemove <T>()
where T : ConcurrencyHandler, new()
{
File.Delete($"concurrencyRemove-{typeof(T).Name}.blob");
var rn = new Random();
using (var s = new BlobStorage($"concurrencyRemove-{typeof(T).Name}.blob"))
{
await s.Initialize <T>(CancellationToken.None);
for (uint j = 0; j < 50; j++)
{
await s.AddChunk(ChunkTypes.Test, j, Enumerable.Range(0, rn.Next(500) + 500).Select(_ => (byte)rn.Next(256)).ToArray(), CancellationToken.None);
}
}
var counter = 0;
const int max = 25;
for (var i = 0; i < max; i++)
{
var t = new Thread(() =>
{
try
{
var r = new Random();
using (var s = new BlobStorage($"concurrencyRemove-{typeof(T).Name}.blob"))
{
s.Initialize <T>(CancellationToken.None).Wait();
for (uint j = 0; j < 40; j++)
{
s.RemoveChunk(si => si.Chunks.Count == 0 ? (StorageChunk?)null : si.Chunks[r.Next(si.Chunks.Count)], CancellationToken.None).Wait();
}
}
}
finally
{
Interlocked.Increment(ref counter);
}
});
t.Start();
Thread.Sleep(100);
}
// ReSharper disable once LoopVariableIsNeverChangedInsideLoop
while (counter < max)
{
Thread.Sleep(100);
}
using (var s = new BlobStorage($"concurrencyRemove-{typeof(T).Name}.blob"))
{
Assert.True(await s.Initialize <T>(CancellationToken.None));
}
}
public async void StreamRead()
{
File.Delete("test.blob");
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
using (var ms = new MemoryStream())
{
// ReSharper disable once AccessToDisposedClosure
var res = await s.ReadChunks(c => c.Id == c1.Id, c => ms, CancellationToken.None);
Assert.Equal(data, ms.ToArray());
Assert.Equal(ms, res.First().Data);
}
using (var ms = new MemoryStream())
{
// ReSharper disable once AccessToDisposedClosure
var res = await s.ReadChunks(sc => sc.Chunks.Where(c => c.Id == c1.Id), c => ms, CancellationToken.None);
Assert.Equal(data, ms.ToArray());
Assert.Equal(ms, res.First().Data);
}
using (var ms = new MemoryStream())
{
// ReSharper disable once AccessToDisposedClosure
var res = await s.ReadChunks(c => c.Id == c1.Id?ms : null, CancellationToken.None);
Assert.Equal(data, ms.ToArray());
Assert.Equal(ms, res.First().Data);
}
using (var ms = new MemoryStream())
{
// ReSharper disable once AccessToDisposedClosure
var res = await s.ReadChunks(sc => sc.Chunks.Where(c => c.Id == c1.Id).Select(c => (c, (Stream)ms)), CancellationToken.None);
Assert.Equal(data, ms.ToArray());
Assert.Equal(ms, res.First().Data);
}
}
}
public async void AddChunk()
{
File.Delete("test.blob");
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
Assert.Equal(1u, c1.Id);
Assert.Equal(11u, c1.UserData);
Assert.Equal((uint)data.Length, c1.Size);
var res = await s.ReadChunks(c => c.Id == 1, CancellationToken.None);
Assert.Equal(data, res.First().Data);
}
}
public async void GetChunks()
{
File.Delete("test.blob");
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
Assert.Empty(await s.GetChunks(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
Assert.Equal(c1, (await s.GetChunks(CancellationToken.None)).Single());
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None);
Assert.Equal(c1.Id, (await s.GetChunks(CancellationToken.None)).Single().Id);
Assert.Equal(ChunkTypes.Free, (await s.GetChunks(CancellationToken.None)).Single().Type);
}
}
public async void RemoveChunk()
{
using (var s = new BlobStorage("test.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
var c1 = await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
Assert.Equal(11u, c1.UserData);
Assert.Equal((uint)data.Length, c1.Size);
var res = await s.ReadChunks(c => c.Id == c1.Id, CancellationToken.None);
Assert.Equal(data, res.First().Data);
await s.RemoveChunk(sc => sc.Chunks.FirstOrDefault(c => c.Id == c1.Id), CancellationToken.None);
var chunks = await s.GetChunks(CancellationToken.None);
Assert.Equal(ChunkTypes.Free, chunks.Single(c => c.Id == c1.Id).Type);
}
}
public async void CrcData()
{
File.Delete("testcrc.blob");
using (var s = new BlobStorage("testcrc.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
var data = Enumerable.Range(0, 256).Select(r => (byte)1).ToArray();
await s.AddChunk(ChunkTypes.Test, 11, data, CancellationToken.None);
}
using (var f = File.OpenWrite("testcrc.blob"))
{
f.Position = 150;
f.WriteByte(5);
}
using (var s = new BlobStorage("testcrc.blob"))
{
Assert.True(await s.Initialize <AppDomainConcurrencyHandler>(CancellationToken.None));
await Assert.ThrowsAsync <InvalidDataException>(() => s.ReadChunks(sc => true, CancellationToken.None));
}
}