Exemplo n.º 1
0
        public static FileHash CalculateHashes(byte[] buffer, long offset, int length, Config config)
        {
            var weakHash = CalculateWeakHash(buffer);
            var strongHash = CalculateStrongHash(buffer);

            var fileHash = new FileHash() { Length = length, Offset = offset, WeakHash = weakHash, StrongHash = strongHash };

            return fileHash;
        }
Exemplo n.º 2
0
        public static ConcurrentBag<FileBlock> GetDiffBlocksForFile(ConcurrentDictionary<long, FileHash> remoteHashes, string filePath, Config config)
        {
            _cancellationSource = new CancellationTokenSource();
            var cancellationToken = _cancellationSource.Token;
            _progressReporter = new ProgressReporter();
            var fileInfo = new FileInfo(filePath);
            var fileChunkSize = Convert.ToInt64(fileInfo.Length / config.DegreeOfParalleism);

            long chunkStart = 0;
            long chunkEnd = 0;
            var resultBag = new ConcurrentBag<FileBlock>();
            using (var mappedFile = MemoryMappedFile.CreateFromFile(filePath))
            {

                var taskArray = new List<Task>();
                int chunkCounter = 0;
                for (var i = 0; i < config.DegreeOfParalleism; i++)
                {
                    chunkEnd = chunkEnd + fileChunkSize;
                    var chunkOverlap = chunkEnd;
                    if (chunkEnd > fileInfo.Length)
                    {
                        chunkEnd = fileInfo.Length;
                        chunkOverlap = chunkEnd;
                    }

                    if (taskArray.Count() <= config.DegreeOfParalleism)
                    {
                        chunkCounter++;
                        var localChunkStart = chunkStart;
                        var length = chunkOverlap - chunkStart;
                        var mappedAccessor = mappedFile.CreateViewAccessor(localChunkStart, length, MemoryMappedFileAccess.Read);
                        var chunkScanTask = Task.Factory.StartNew(
                            () => ScanFileChunk(chunkCounter, cancellationToken, remoteHashes, mappedAccessor, localChunkStart, length, config), cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default)
                            .ContinueWith(
                                outHash => outHash.Result.ForEach(resultBag.Add));

                        taskArray.Add(chunkScanTask);

                    }

                    Interlocked.Add(ref chunkStart, fileChunkSize);

                }
                Task.WaitAll(taskArray.ToArray());
            }
            return resultBag;
        }
Exemplo n.º 3
0
        public static ConcurrentDictionary<long, FileHash> GetHashesForFile(string filePath, Config config)
        {
            var resultDictionary = new ConcurrentDictionary<long, FileHash>();
            if (!File.Exists(filePath)) throw new NullReferenceException("File cannot be found.");

            var fileInfo = new FileInfo(filePath);

            long fileLength = fileInfo.Length;

            int localBlockLength = config.BlockLength;
            if (fileLength <= localBlockLength)
            {
                localBlockLength = checked((int)fileLength);
            }

            long currentFilePosition = 0;
            var bytesToRead = config.BlockLength;
            var buffer = new byte[bytesToRead];
            var procCount = config.DegreeOfParalleism * 15;
            var taskArray = new List<Task>(procCount);
            using (var file = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                while (currentFilePosition < fileLength)
                {
                    if (currentFilePosition + localBlockLength > fileLength)
                    {
                        bytesToRead = checked((int)(fileLength - currentFilePosition));
                        buffer = new byte[bytesToRead];
                    }
                    var offSetPosition = currentFilePosition;

                    if (file.Read(buffer, 0, bytesToRead) < 0)
                    {
                        break;
                    }

                    currentFilePosition = file.Position;

                    if (taskArray.Count <= procCount)
                    {
                        taskArray.Add(Task.Factory.StartNew(() => CalculateHashes(buffer, offSetPosition, bytesToRead, config))
                                          .ContinueWith(
                                              outHash => resultDictionary.TryAdd(outHash.Result.WeakHash, outHash.Result)));

                    }
                    else
                    {
                        var complete = Task.WaitAny(taskArray.ToArray());
                        taskArray.RemoveAt(complete);
                    }

                }

            }
            Task.WaitAll(taskArray.ToArray());

            return resultDictionary;
        }
Exemplo n.º 4
0
        public static List<FileBlock> ScanFileChunk(int blockNumber, CancellationToken cancellationToken, 
            ConcurrentDictionary<long, FileHash> remoteHashes, MemoryMappedViewAccessor chunkMap, long chunkStartOffset, long chunkLength, Config config)
        {
            var bytesToRead = config.BlockLength;

            var outList = new List<FileBlock>();
            long readStartOffset = 0;
            var nonMatchStartOffset = readStartOffset;
            var windowChecksum = new Adler32();

            while (readStartOffset < chunkLength)
            {
                if (cancellationToken.IsCancellationRequested)
                    return null;

                var buffer = new byte[bytesToRead];

                if (readStartOffset + config.BlockLength > chunkLength)
                {
                    bytesToRead = checked((int)(chunkLength - readStartOffset));
                    buffer = new byte[bytesToRead];
                }

                chunkMap.ReadArray((int)readStartOffset, buffer, 0, buffer.Length);

                if (readStartOffset == 0)
                {
                    windowChecksum.Update(buffer);
                }
                else
                {
                    windowChecksum.Update(buffer, buffer.Length - 1, 1);
                }

                var weakCheckSum = windowChecksum.Value;

                FileHash match;
                remoteHashes.TryGetValue(weakCheckSum, out match);
                if (match != null)
                {
                    var strongHash = CalculateStrongHash(buffer);
                    if (strongHash.SequenceEqual(match.StrongHash))
                    {
                        if (readStartOffset > 0)
                        {
                            long nonMatchEndOffset = readStartOffset - 1;
                            var nonMatchingChunk = new FileBlock
                                                       {
                                                           IsMatch = false,
                                                           BlockLength = nonMatchEndOffset - nonMatchStartOffset,
                                                           SourceOffset = nonMatchStartOffset + readStartOffset
                                                       };
                            outList.Add(nonMatchingChunk);
                            nonMatchStartOffset = readStartOffset + config.BlockLength;
                        }

                        var matchingChunk = new FileBlock
                                                {
                                                    IsMatch = true,
                                                    DestinationOffset = chunkStartOffset + readStartOffset,
                                                    SourceOffset = match.Offset,
                                                    BlockLength = config.BlockLength
                                                };
                        outList.Add(matchingChunk);
                        windowChecksum.Reset();
                    }
                }
                readStartOffset += 1;

                if (readStartOffset % 100 == 0)
                {

                    _progressReporter.ReportProgress(()=>
                                                         {

                                                         })
                    ;
                }
            }
            if (chunkLength - nonMatchStartOffset > 1)
            {
                var nonMatchingChunk = new FileBlock() { IsMatch = false, BlockLength = chunkLength - nonMatchStartOffset, SourceOffset = nonMatchStartOffset + readStartOffset };
                outList.Add(nonMatchingChunk);
            }
            return outList;
        }