예제 #1
0
        private static IBlockManagingStore CreateDedupStore(string dataStoreDir)
        {
            const uint BlockSize = blockSize;
            long StoreSize = 6 * 1024 * 1024 * 1024L;
            if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
            {
                StoreSize = 50 * 1024 * 1024 * 1024L;
            }
            if (Directory.Exists(dataStoreDir))
            {
                Directory.Delete(dataStoreDir, true);
            }
            Directory.CreateDirectory(dataStoreDir);
            var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
            xRawBlockStoreFS.SetLength(StoreSize);
            var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
            //var xRawBlockStoreCache = new ExperimentalReadWriteCachingBlockStore(xRawBlockStore, BlockSize);
            //var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
            //xRawBlockStoreFS.SetLength(StoreSize);
            //var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
            //var xRawBlockStoreCache = new ReadCachingBlockStore(xRawBlockStore, 128 * BlockSize);

            var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Create);
            xVirtualBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));

            var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Create);
            xVirtualBlockStoreFS.SetLength(StoreSize);
            var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);
            var xVirtualBlockStoreCache = new SmarterReadCachingBlockStore(xVirtualBlockStore, BlockSize);

            var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Create);
            xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
            var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Create);
            xRawBlockUsageCounterFS.SetLength((long)(xRawBlockStore.BlockCount * 8));
            var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
            var xRawBlockUsageCounterStoreCache = new SimpleReadWriteCachingBlockStore(xRawBlockUsageCounterStore, 1024);
            var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStoreCache);

            SimpleHashManager.Create(Path.Combine(dataStoreDir, "Hashes"));
            var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));

            var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;

            //return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
            //    , Path.Combine(dataStoreDir, "BatchCache"));
            return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
        }
예제 #2
0
 public void Test_ChecksDisposeWhenDisposed()
 {
     using (var xMSStore = new MemoryStream(new byte[32768]))
     {
         using (var xMSBitmap = new MemoryStream(new byte[8]))
         {
             using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
             {
                 var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8);
                 xBitmapBlockManager.Dispose();
                 xBitmapBlockManager.Dispose();
             }
         }
     }
 }
예제 #3
0
 public void DoTestMarkReserve()
 {
     using (var xMSStore = new MemoryStream(new byte[32768]))
     {
         using (var xMSBitmap = new MemoryStream(new byte[8]))
         {
             using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
             {
                 using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
                 {
                     Assert.IsFalse(xBitmapBlockManager.IsReserved(0));
                     var xBit = xBitmapBlockManager.Reserve();
                     Assert.IsTrue(xBitmapBlockManager.IsReserved(xBit));
                     xBitmapBlockManager.Free(xBit);
                     Assert.IsFalse(xBitmapBlockManager.IsReserved(xBit));
                 }
             }
         }
     }
 }
        private DeduplicatingBlockStore CreateStore(uint blockSize, ulong virtualBlockCount, ulong rawBlockCount)
        {
            var xBaseDir = Path.Combine(Environment.CurrentDirectory, StoreSubdir);
            if (Directory.Exists(xBaseDir))
            {
                Directory.Delete(xBaseDir, true);
            }
            Directory.CreateDirectory(xBaseDir);

            var xRawStoreSize = rawBlockCount * blockSize;

            var xRawBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "RawBlocks.bin"), FileMode.CreateNew);
            xRawBlockStoreFS.SetLength((long)xRawStoreSize);
            var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, blockSize);

            var xRawBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "RawBlockBitmap.bin"), FileMode.CreateNew);
            xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
            mRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / blockSize), blockSize);

            var xVirtualBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocksBitmap.bin"), FileMode.CreateNew);
            xVirtualBlockManagerFS.SetLength((long)(virtualBlockCount /8 ));
            var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, virtualBlockCount / blockSize / 8, blockSize);

            var xVirtualBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocks.bin"), FileMode.CreateNew);
            xVirtualBlockStoreFS.SetLength((long)(virtualBlockCount * 8));
            var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);

            var xRawBlockUsageCounterFS = new FileStream(Path.Combine(xBaseDir, "RawBlockUsageCounts.bin"), FileMode.CreateNew);
            xRawBlockUsageCounterFS.SetLength((long)(rawBlockCount * 8));
            var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, blockSize);
            var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);

            var xHashesDir = Path.Combine(xBaseDir, "Hashes");
            SimpleHashManager.Create(xHashesDir);
            var xHashManager = new SimpleHashManager(xHashesDir);

            return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, mRawBlockManager, virtualBlockCount,
                xRawBlockUsageCounter, xHashManager);
        }
예제 #5
0
 public void Test_MarkReservedAfterFullFillAndSomeFree()
 {
     using (var xMSBitmap = new MemoryStream(new byte[8]))
     {
         using (var xBitmap = new BitmapBlockManager(xMSBitmap, 1, 8))
         {
             var xReservations = xBitmap.Reserve(64);
             Assert.AreEqual(64, xReservations.Length);
             ulong xReservation;
             Assert.IsFalse(xBitmap.TryReserve(out xReservation));
             xBitmap.Free(xReservations[0]);
             Assert.IsTrue(xBitmap.TryReserve(out xReservation));
             Assert.IsFalse(xBitmap.TryReserve(out xReservation));
         }
     }
 }
예제 #6
0
 public void Test_ChecksForBitmapSize()
 {
     using (var xMSStore = new MemoryStream(new byte[32768]))
     {
         using (var xMSBitmap = new MemoryStream(new byte[7]))
         {
             using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
             {
                 using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
                 {
                 }
             }
         }
     }
 }
예제 #7
0
 public void Test_ThrowErrorWithEmptyBitmapStream()
 {
     using (var xMSStore = new MemoryStream(new byte[32768]))
     {
         using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
         {
             using (var xBitmapBlockManager = new BitmapBlockManager(xBlockStore, null))
             {
             }
         }
     }
 }
예제 #8
0
 public void Test_ThrowErrorWithEmptyBaseStore()
 {
     using (var xMSBitmap = new MemoryStream(new byte[8]))
     {
         using (var xBitmapBlockManager = new BitmapBlockManager(null, xMSBitmap))
         {
         }
     }
 }
예제 #9
0
 public void Test_MultipleReserve()
 {
     using (var xMSStore = new MemoryStream(new byte[32768]))
     {
         using (var xMSBitmap = new MemoryStream(new byte[8]))
         {
             using (var xBlockStore = new SimpleStreamBlockStore(xMSStore, 512))
             {
                 using (var xBitmapBlockManager = new BitmapBlockManager(xMSBitmap, 1, 8))
                 {
                     var xResult = xBitmapBlockManager.Reserve(3);
                     Assert.AreEqual(3, xResult.Length);
                     // todo: how to test better?
                 }
             }
         }
     }
 }
예제 #10
0
        private static IBlockManagingStore OpenDedupStore(string dataStoreDir)
        {
            // for now hardcode these values
            const uint BlockSize = blockSize;
            long StoreSize = 6 * 1024 * 1024 * 1024L;
            if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
            {
                StoreSize = 50 * 1024 * 1024 * 1024L;
            }
            var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Open);
            var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);

            var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Open);
            if (xVirtualBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
            {
                throw new Exception("VirtualBlockBitmap.bin file size mismatch!");
            }

            var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Open);
            if (xVirtualBlockStoreFS.Length != StoreSize)
            {
                throw new Exception("VirtualBlocks.bin file size mismatch!");
            }
            var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, BlockSize);

            var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Open);
            if (xRawBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
            {
                throw new Exception("RawBlockBitmap.bin file size mismatch!");
            }
            var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Open);
            if (xRawBlockUsageCounterFS.Length != (long)(xRawBlockStore.BlockCount * 8))
            {
                throw new Exception("RawBlockUsageCounts.bin file size mismatch!");
            }
            var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
            var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);

            var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));

            var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;

            //return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
            //    , Path.Combine(dataStoreDir, "BatchCache"));
            return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
        }