public void TestTwoBlocks()
{
using (var xMemStream = new MemoryStream())
{
xMemStream.SetLength(2048);
// 4 blocks of 512 bytes
using (var xRawBlockStore = new SimpleStreamBlockStore(xMemStream, 512))
{
ODBFSImpl.Format(xRawBlockStore, 512);
using (var xOdbfs = new ODBFSImpl(xRawBlockStore))
{
Assert.AreEqual(0, xOdbfs.GetVirtualBlocks().Count());
var xBlock1Id = Guid.NewGuid();
var xBlock2Id = Guid.NewGuid();
xOdbfs.CreateNewBlock(xBlock1Id, 1);
xOdbfs.CreateNewBlock(xBlock2Id, 1);
var xBuff1 = new byte[512];
var xBuff1Seg = new ArraySegment<byte>(xBuff1);
var xBuff2 = new byte[512];
var xBuff2Seg = new ArraySegment<byte>(xBuff2);
using (var xBlock1 = xOdbfs.OpenBlock(xBlock1Id))
{
for (int i = 0; i < xBuff1.Length; i++)
{
xBuff1[i] = 0xAA;
}
xBlock1.Store(0, xBuff1Seg);
}
using (var xBlock2 = xOdbfs.OpenBlock(xBlock2Id))
{
for (int i = 0; i < xBuff1.Length; i++)
{
xBuff1[i] = 0xBB;
}
xBlock2.Store(0, xBuff1Seg);
}
for (int i = 0; i < xBuff1.Length; i++)
{
xBuff1[i] = 0xAA;
}
xRawBlockStore.Retrieve(1, xBuff2Seg);
Assert.IsTrue(DeduplicatingBlockStore.CompareBlocks(xBuff1, xBuff2, 0, 0, 512));
for (int i = 0; i < xBuff1.Length; i++)
{
xBuff1[i] = 0xBB;
}
xRawBlockStore.Retrieve(2, xBuff2Seg);
Assert.IsTrue(DeduplicatingBlockStore.CompareBlocks(xBuff1, xBuff2, 0, 0, 512));
xBuff1 = new byte[512];
xRawBlockStore.Retrieve(3, xBuff2Seg);
Assert.IsTrue(DeduplicatingBlockStore.CompareBlocks(xBuff1, xBuff2, 0, 0, 512));
}
}
}
}
public void Test_StoreThrowsErrorAfterDisposed()
{
var xBuff = new byte[2];
var xSeg = new ArraySegment<byte>(xBuff);
var xBlock = new SimpleStreamBlockStore(new MemoryStream(new byte[2]), 2);
using (xBlock)
{
xBlock.Retrieve(0, xSeg);
}
xBlock.Retrieve(0, xSeg);
}
private static IBlockManagingStore CreateDedupStore(string dataStoreDir)
{
const uint BlockSize = blockSize;
long StoreSize = 6 * 1024 * 1024 * 1024L;
if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
{
StoreSize = 50 * 1024 * 1024 * 1024L;
}
if (Directory.Exists(dataStoreDir))
{
Directory.Delete(dataStoreDir, true);
}
Directory.CreateDirectory(dataStoreDir);
var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
xRawBlockStoreFS.SetLength(StoreSize);
var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
//var xRawBlockStoreCache = new ExperimentalReadWriteCachingBlockStore(xRawBlockStore, BlockSize);
//var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
//xRawBlockStoreFS.SetLength(StoreSize);
//var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
//var xRawBlockStoreCache = new ReadCachingBlockStore(xRawBlockStore, 128 * BlockSize);
var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Create);
xVirtualBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Create);
xVirtualBlockStoreFS.SetLength(StoreSize);
var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);
var xVirtualBlockStoreCache = new SmarterReadCachingBlockStore(xVirtualBlockStore, BlockSize);
var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Create);
xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Create);
xRawBlockUsageCounterFS.SetLength((long)(xRawBlockStore.BlockCount * 8));
var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
var xRawBlockUsageCounterStoreCache = new SimpleReadWriteCachingBlockStore(xRawBlockUsageCounterStore, 1024);
var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStoreCache);
SimpleHashManager.Create(Path.Combine(dataStoreDir, "Hashes"));
var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));
var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;
//return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
// , Path.Combine(dataStoreDir, "BatchCache"));
return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
}
public void Test_ChecksDisposeWhenDisposed()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8);
xBitmapBlockManager.Dispose();
xBitmapBlockManager.Dispose();
}
}
}
}
public void DoTestMarkReserve()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
{
Assert.IsFalse(xBitmapBlockManager.IsReserved(0));
var xBit = xBitmapBlockManager.Reserve();
Assert.IsTrue(xBitmapBlockManager.IsReserved(xBit));
xBitmapBlockManager.Free(xBit);
Assert.IsFalse(xBitmapBlockManager.IsReserved(xBit));
}
}
}
}
}
public void TestWithFileStream()
{
var xTempFile = Path.GetTempFileName();
try
{
using (var xFS = new FileStream(xTempFile, FileMode.Open))
{
xFS.SetLength(2);
}
using (var xStore = new SimpleStreamBlockStore(xTempFile, 2))
{
Assert.AreEqual(2u, xStore.BlockSize);
Assert.AreEqual(1ul, xStore.BlockCount);
}
}
finally
{
File.Delete(xTempFile);
}
}
private DeduplicatingBlockStore CreateStore(uint blockSize, ulong virtualBlockCount, ulong rawBlockCount)
{
var xBaseDir = Path.Combine(Environment.CurrentDirectory, StoreSubdir);
if (Directory.Exists(xBaseDir))
{
Directory.Delete(xBaseDir, true);
}
Directory.CreateDirectory(xBaseDir);
var xRawStoreSize = rawBlockCount * blockSize;
var xRawBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "RawBlocks.bin"), FileMode.CreateNew);
xRawBlockStoreFS.SetLength((long)xRawStoreSize);
var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, blockSize);
var xRawBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "RawBlockBitmap.bin"), FileMode.CreateNew);
xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
mRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / blockSize), blockSize);
var xVirtualBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocksBitmap.bin"), FileMode.CreateNew);
xVirtualBlockManagerFS.SetLength((long)(virtualBlockCount /8 ));
var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, virtualBlockCount / blockSize / 8, blockSize);
var xVirtualBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocks.bin"), FileMode.CreateNew);
xVirtualBlockStoreFS.SetLength((long)(virtualBlockCount * 8));
var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);
var xRawBlockUsageCounterFS = new FileStream(Path.Combine(xBaseDir, "RawBlockUsageCounts.bin"), FileMode.CreateNew);
xRawBlockUsageCounterFS.SetLength((long)(rawBlockCount * 8));
var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, blockSize);
var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);
var xHashesDir = Path.Combine(xBaseDir, "Hashes");
SimpleHashManager.Create(xHashesDir);
var xHashManager = new SimpleHashManager(xHashesDir);
return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, mRawBlockManager, virtualBlockCount,
xRawBlockUsageCounter, xHashManager);
}
public void DoTest()
{
using (var xBackend = new MemoryStream())
{
xBackend.SetLength(2);
var xStore = new SimpleStreamBlockStore(xBackend, 1);
Assert.AreEqual(1u, xStore.BlockSize);
Assert.AreEqual(2u, xStore.BlockCount);
var xBuff = new byte[1];
xBuff[0] = 1;
var xArraySegment = new ArraySegment<byte>(xBuff);
xStore.Retrieve(0, xArraySegment);
Assert.AreEqual(0, xBuff[0]);
// now store a byte in block 0, and read back both 0 and 1
xBuff[0] = 2;
xStore.Store(1, xArraySegment);
xStore.Retrieve(0, xArraySegment);
Assert.AreEqual(0, xBuff[0]);
xStore.Retrieve(1, xArraySegment);
Assert.AreEqual(2, xBuff[0]);
}
}
public void Test_ChecksForBitmapSize()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[7]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
{
}
}
}
}
}
public void Test_ThrowErrorWithEmptyBitmapStream()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xBlockStore, null))
{
}
}
}
}
public void Test_MultipleReserve()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
{
var xResult = xBitmapBlockManager.Reserve(3);
Assert.AreEqual(3, xResult.Length);
// todo: how to test better?
}
}
}
}
}
public void Test_StoreThrowsErrorWithWrongBufferSizeInRetrieve()
{
using (var xMS = new MemoryStream(new byte[2]))
{
IBlockStore xStore = new SimpleStreamBlockStore(xMS, 2);
var xBuff = new byte[1];
var xSeg = new ArraySegment<byte>(xBuff);
xStore.Retrieve(0, xSeg);
}
}
public void Test_StoreThrowsErrorWithoutBackend()
{
IBlockStore xStore = new SimpleStreamBlockStore(null as Stream, 1);
}
public void Test_StoreThrowsErrorWithWrongSize()
{
using (var xMS = new MemoryStream(new byte[2]))
{
var xStore = new SimpleStreamBlockStore(xMS, 4);
}
}
private static IBlockManagingStore OpenDedupStore(string dataStoreDir)
{
// for now hardcode these values
const uint BlockSize = blockSize;
long StoreSize = 6 * 1024 * 1024 * 1024L;
if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
{
StoreSize = 50 * 1024 * 1024 * 1024L;
}
var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Open);
var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Open);
if (xVirtualBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
{
throw new Exception("VirtualBlockBitmap.bin file size mismatch!");
}
var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Open);
if (xVirtualBlockStoreFS.Length != StoreSize)
{
throw new Exception("VirtualBlocks.bin file size mismatch!");
}
var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, BlockSize);
var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Open);
if (xRawBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
{
throw new Exception("RawBlockBitmap.bin file size mismatch!");
}
var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Open);
if (xRawBlockUsageCounterFS.Length != (long)(xRawBlockStore.BlockCount * 8))
{
throw new Exception("RawBlockUsageCounts.bin file size mismatch!");
}
var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);
var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));
var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;
//return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
// , Path.Combine(dataStoreDir, "BatchCache"));
return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
}