Пример #1
0
        private static IBlockManagingStore CreateDedupStore(string dataStoreDir)
        {
            const uint BlockSize = blockSize;
            long StoreSize = 6 * 1024 * 1024 * 1024L;
            if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
            {
                StoreSize = 50 * 1024 * 1024 * 1024L;
            }
            if (Directory.Exists(dataStoreDir))
            {
                Directory.Delete(dataStoreDir, true);
            }
            Directory.CreateDirectory(dataStoreDir);
            var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
            xRawBlockStoreFS.SetLength(StoreSize);
            var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
            //var xRawBlockStoreCache = new ExperimentalReadWriteCachingBlockStore(xRawBlockStore, BlockSize);
            //var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Create);
            //xRawBlockStoreFS.SetLength(StoreSize);
            //var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);
            //var xRawBlockStoreCache = new ReadCachingBlockStore(xRawBlockStore, 128 * BlockSize);

            var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Create);
            xVirtualBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));

            var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Create);
            xVirtualBlockStoreFS.SetLength(StoreSize);
            var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);
            var xVirtualBlockStoreCache = new SmarterReadCachingBlockStore(xVirtualBlockStore, BlockSize);

            var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Create);
            xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
            var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Create);
            xRawBlockUsageCounterFS.SetLength((long)(xRawBlockStore.BlockCount * 8));
            var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
            var xRawBlockUsageCounterStoreCache = new SimpleReadWriteCachingBlockStore(xRawBlockUsageCounterStore, 1024);
            var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStoreCache);

            SimpleHashManager.Create(Path.Combine(dataStoreDir, "Hashes"));
            var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));

            var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;

            //return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
            //    , Path.Combine(dataStoreDir, "BatchCache"));
            return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
        }
        private DeduplicatingBlockStore CreateStore(uint blockSize, ulong virtualBlockCount, ulong rawBlockCount)
        {
            var xBaseDir = Path.Combine(Environment.CurrentDirectory, StoreSubdir);
            if (Directory.Exists(xBaseDir))
            {
                Directory.Delete(xBaseDir, true);
            }
            Directory.CreateDirectory(xBaseDir);

            var xRawStoreSize = rawBlockCount * blockSize;

            var xRawBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "RawBlocks.bin"), FileMode.CreateNew);
            xRawBlockStoreFS.SetLength((long)xRawStoreSize);
            var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, blockSize);

            var xRawBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "RawBlockBitmap.bin"), FileMode.CreateNew);
            xRawBlockManagerFS.SetLength((long)(xRawBlockStore.BlockCount / 8));
            mRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / blockSize), blockSize);

            var xVirtualBlockManagerFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocksBitmap.bin"), FileMode.CreateNew);
            xVirtualBlockManagerFS.SetLength((long)(virtualBlockCount /8 ));
            var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, virtualBlockCount / blockSize / 8, blockSize);

            var xVirtualBlockStoreFS = new FileStream(Path.Combine(xBaseDir, "VirtualBlocks.bin"), FileMode.CreateNew);
            xVirtualBlockStoreFS.SetLength((long)(virtualBlockCount * 8));
            var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, 8);

            var xRawBlockUsageCounterFS = new FileStream(Path.Combine(xBaseDir, "RawBlockUsageCounts.bin"), FileMode.CreateNew);
            xRawBlockUsageCounterFS.SetLength((long)(rawBlockCount * 8));
            var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, blockSize);
            var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);

            var xHashesDir = Path.Combine(xBaseDir, "Hashes");
            SimpleHashManager.Create(xHashesDir);
            var xHashManager = new SimpleHashManager(xHashesDir);

            return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, mRawBlockManager, virtualBlockCount,
                xRawBlockUsageCounter, xHashManager);
        }
Пример #3
0
        private static IBlockManagingStore OpenDedupStore(string dataStoreDir)
        {
            // for now hardcode these values
            const uint BlockSize = blockSize;
            long StoreSize = 6 * 1024 * 1024 * 1024L;
            if (Environment.MachineName.Equals("tw-vms1", StringComparison.InvariantCultureIgnoreCase))
            {
                StoreSize = 50 * 1024 * 1024 * 1024L;
            }
            var xRawBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "RawBlocks.bin"), FileMode.Open);
            var xRawBlockStore = new SimpleStreamBlockStore(xRawBlockStoreFS, BlockSize);

            var xVirtualBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlockBitmap.bin"), FileMode.Open);
            if (xVirtualBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
            {
                throw new Exception("VirtualBlockBitmap.bin file size mismatch!");
            }

            var xVirtualBlockManager = new BitmapBlockManager(xVirtualBlockManagerFS, (ulong)(xVirtualBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xVirtualBlockStoreFS = new FileStream(Path.Combine(dataStoreDir, "VirtualBlocks.bin"), FileMode.Open);
            if (xVirtualBlockStoreFS.Length != StoreSize)
            {
                throw new Exception("VirtualBlocks.bin file size mismatch!");
            }
            var xVirtualBlockStore = new SimpleStreamBlockStore(xVirtualBlockStoreFS, BlockSize);

            var xRawBlockManagerFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockBitmap.bin"), FileMode.Open);
            if (xRawBlockManagerFS.Length != (long)(xRawBlockStore.BlockCount / 8))
            {
                throw new Exception("RawBlockBitmap.bin file size mismatch!");
            }
            var xRawBlockManager = new BitmapBlockManager(xRawBlockManagerFS, (ulong)(xRawBlockManagerFS.Length / BlockSize), xRawBlockStore.BlockSize);

            var xRawBlockUsageCounterFS = new FileStream(Path.Combine(dataStoreDir, "RawBlockUsageCounts.bin"), FileMode.Open);
            if (xRawBlockUsageCounterFS.Length != (long)(xRawBlockStore.BlockCount * 8))
            {
                throw new Exception("RawBlockUsageCounts.bin file size mismatch!");
            }
            var xRawBlockUsageCounterStore = new SimpleStreamBlockStore(xRawBlockUsageCounterFS, BlockSize);
            var xRawBlockUsageCounter = new SimpleUsageCountStore(xRawBlockUsageCounterStore);

            var xHashManager = new SimpleHashManager(Path.Combine(dataStoreDir, "Hashes"));

            var xVirtualBlockCount = (xVirtualBlockStore.BlockCount * xVirtualBlockStore.BlockSize) / 8;

            //return new ExperimentalDeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStoreCache, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager
            //    , Path.Combine(dataStoreDir, "BatchCache"));
            return new DeduplicatingBlockStore(xVirtualBlockManager, xVirtualBlockStore, xRawBlockStore, xRawBlockManager, xVirtualBlockCount, xRawBlockUsageCounter, xHashManager);
        }