private static IBlockManagingStore CreateDedupStore(string dataStoreDir)
{
const uint BlockSize = blockSize;
long StoreSize = 6 * 1024 * 1024 * 1024L;
if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
{
StoreSize = 50 * 1024 * 1024 * 1024L;
}
if (Directory.Exists(dataStoreDir))
{
Directory.Delete(dataStoreDir, true);
}
Directory.CreateDirectory(dataStoreDir);
var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
xRawBlockStoreFS.SetLength(StoreSize);
var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
//var xRawBlockStoreCache = new ExperimentalReadWriteCachingBlockStore(xRawBlockStore, BlockSize);
//var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
//xRawBlockStoreFS.SetLength(StoreSize);
//var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
//var xRawBlockStoreCache = new ReadCachingBlockStore(xRawBlockStore, 128 * BlockSize);
var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Create);
xVirtualBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Create);
xVirtualBlockStoreFS.SetLength(StoreSize);
var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);
var xVirtualBlockStoreCache = new SmarterReadCachingBlockStore(xVirtualBlockStore, BlockSize);
var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Create);
xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Create);
xRawBlockUsageCounterFS.SetLength((long)(xRawBlockStore.BlockCount * 8));
var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
var xRawBlockUsageCounterStoreCache = new SimpleReadWriteCachingBlockStore(xRawBlockUsageCounterStore, 1024);
var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStoreCache);
SimpleHashManager.Create(Path.Combine(dataStoreDir, "Hashes"));
var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));
var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;
//return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
// , Path.Combine(dataStoreDir, "BatchCache"));
return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
}
public void Test_ChecksDisposeWhenDisposed()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8);
xBitmapBlockManager.Dispose();
xBitmapBlockManager.Dispose();
}
}
}
}
public void DoTestMarkReserve()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
{
Assert.IsFalse(xBitmapBlockManager.IsReserved(0));
var xBit = xBitmapBlockManager.Reserve();
Assert.IsTrue(xBitmapBlockManager.IsReserved(xBit));
xBitmapBlockManager.Free(xBit);
Assert.IsFalse(xBitmapBlockManager.IsReserved(xBit));
}
}
}
}
}
private DeduplicatingBlockStore CreateStore(uint blockSize, ulong virtualBlockCount, ulong rawBlockCount)
{
var xBaseDir = Path.Combine(Environment.CurrentDirectory, StoreSubdir);
if (Directory.Exists(xBaseDir))
{
Directory.Delete(xBaseDir, true);
}
Directory.CreateDirectory(xBaseDir);
var xRawStoreSize = rawBlockCount * blockSize;
var xRawBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "RawBlocks.bin"), FileMode.CreateNew);
xRawBlockStoreFS.SetLength((long)xRawStoreSize);
var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, blockSize);
var xRawBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "RawBlockBitmap.bin"), FileMode.CreateNew);
xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
mRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / blockSize), blockSize);
var xVirtualBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocksBitmap.bin"), FileMode.CreateNew);
xVirtualBlockManagerFS.SetLength((long)(virtualBlockCount /8 ));
var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, virtualBlockCount / blockSize / 8, blockSize);
var xVirtualBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocks.bin"), FileMode.CreateNew);
xVirtualBlockStoreFS.SetLength((long)(virtualBlockCount * 8));
var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);
var xRawBlockUsageCounterFS = new FileStream(Path.Combine(xBaseDir, "RawBlockUsageCounts.bin"), FileMode.CreateNew);
xRawBlockUsageCounterFS.SetLength((long)(rawBlockCount * 8));
var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, blockSize);
var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);
var xHashesDir = Path.Combine(xBaseDir, "Hashes");
SimpleHashManager.Create(xHashesDir);
var xHashManager = new SimpleHashManager(xHashesDir);
return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, mRawBlockManager, virtualBlockCount,
xRawBlockUsageCounter, xHashManager);
}
public void Test_MarkReservedAfterFullFillAndSomeFree()
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBitmap = new BitmapBlockManager(xMSBitmap, 1, 8))
{
var xReservations = xBitmap.Reserve(64);
Assert.AreEqual(64, xReservations.Length);
ulong xReservation;
Assert.IsFalse(xBitmap.TryReserve(out xReservation));
xBitmap.Free(xReservations[0]);
Assert.IsTrue(xBitmap.TryReserve(out xReservation));
Assert.IsFalse(xBitmap.TryReserve(out xReservation));
}
}
}
public void Test_ChecksForBitmapSize()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[7]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
{
}
}
}
}
}
public void Test_ThrowErrorWithEmptyBitmapStream()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xBlockStore, null))
{
}
}
}
}
public void Test_ThrowErrorWithEmptyBaseStore()
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBitmapBlockManager = new BitmapBlockManager(null, xMSBitmap))
{
}
}
}
public void Test_MultipleReserve()
{
using (var xMSStore = new MemoryStream(new byte[32768]))
{
using (var xMSBitmap = new MemoryStream(new byte[8]))
{
using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
{
using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
{
var xResult = xBitmapBlockManager.Reserve(3);
Assert.AreEqual(3, xResult.Length);
// todo: how to test better?
}
}
}
}
}
private static IBlockManagingStore OpenDedupStore(string dataStoreDir)
{
// for now hardcode these values
const uint BlockSize = blockSize;
long StoreSize = 6 * 1024 * 1024 * 1024L;
if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
{
StoreSize = 50 * 1024 * 1024 * 1024L;
}
var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Open);
var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Open);
if (xVirtualBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
{
throw new Exception("VirtualBlockBitmap.bin file size mismatch!");
}
var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Open);
if (xVirtualBlockStoreFS.Length != StoreSize)
{
throw new Exception("VirtualBlocks.bin file size mismatch!");
}
var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, BlockSize);
var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Open);
if (xRawBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
{
throw new Exception("RawBlockBitmap.bin file size mismatch!");
}
var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);
var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Open);
if (xRawBlockUsageCounterFS.Length != (long)(xRawBlockStore.BlockCount * 8))
{
throw new Exception("RawBlockUsageCounts.bin file size mismatch!");
}
var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);
var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));
var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;
//return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
// , Path.Combine(dataStoreDir, "BatchCache"));
return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
}