private List <DedupeChunk> ChunkStream(string key, long contentLength, Stream stream, Action <DedupeChunk, DedupeObjectMap> processChunk)
        {
            if (String.IsNullOrEmpty(key))
            {
                throw new ArgumentNullException(nameof(key));
            }
            if (contentLength < 1)
            {
                throw new ArgumentException("Content length must be greater than zero.");
            }
            if (stream == null)
            {
                throw new ArgumentNullException(nameof(stream));
            }
            if (!stream.CanRead)
            {
                throw new ArgumentException("Cannot read from supplied stream.");
            }
            if (processChunk == null)
            {
                throw new ArgumentNullException(nameof(processChunk));
            }

            #region Initialize

            List <DedupeChunk> chunks = new List <DedupeChunk>();
            DedupeObjectMap    map    = null;
            DedupeChunk        chunk  = null;
            long   bytesRead          = 0;
            string chunkKey           = null;

            #endregion

            if (contentLength <= _Settings.MinChunkSize)
            {
                #region Single-Chunk

                byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
                chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
                chunk    = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
                chunks.Add(chunk);

                map = new DedupeObjectMap(key, chunkKey, chunk.Length, 0, 0);
                processChunk(chunk, map);
                return(chunks);

                #endregion
            }
            else
            {
                #region Sliding-Window

                Streams streamWindow = new Streams(stream, contentLength, _Settings.MinChunkSize, _Settings.ShiftCount);
                byte[]  chunkData    = null;
                long    chunkAddress = 0;  // should only be set at the beginning of a new chunk

                while (true)
                {
                    byte[] newData    = null;
                    bool   finalChunk = false;

                    long   tempPosition = 0;
                    byte[] window       = streamWindow.GetNextChunk(out tempPosition, out newData, out finalChunk);
                    if (window == null)
                    {
                        return(chunks);
                    }
                    if (chunkData == null)
                    {
                        chunkAddress = tempPosition;
                    }

                    if (chunkData == null)
                    {
                        // starting a new chunk
                        chunkData = new byte[window.Length];
                        Buffer.BlockCopy(window, 0, chunkData, 0, window.Length);
                    }
                    else
                    {
                        // append new data
                        chunkData = DedupeCommon.AppendBytes(chunkData, newData);
                    }

                    byte[] md5Hash = DedupeCommon.Md5(window);
                    if (DedupeCommon.IsZeroBytes(md5Hash, _Settings.BoundaryCheckBytes) ||
                        chunkData.Length >= _Settings.MaxChunkSize)
                    {
                        #region Chunk-Boundary

                        chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));

                        chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
                        map   = new DedupeObjectMap(key, chunk.Key, chunkData.Length, chunks.Count, chunkAddress);
                        processChunk(chunk, map);
                        chunk.Data = null;
                        chunks.Add(chunk);

                        chunk     = null;
                        chunkData = null;

                        streamWindow.AdvanceToNewChunk();

                        #endregion
                    }
                    else
                    {
                        // do nothing, continue;
                    }

                    if (finalChunk)
                    {
                        #region Final-Chunk

                        if (chunkData != null)
                        {
                            chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
                            chunk    = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
                            map      = new DedupeObjectMap(key, chunk.Key, chunk.Length, chunks.Count, chunkAddress);
                            processChunk(chunk, map);
                            chunk.Data = null;
                            chunks.Add(chunk);
                            break;
                        }

                        #endregion
                    }
                }

                #endregion
            }

            return(chunks);
        }
Exemple #2
0
        private bool ChunkStream(long contentLength, Stream stream, Func <Chunk, bool> processChunk, out List <Chunk> chunks)
        {
            #region Initialize

            chunks = new List <Chunk>();
            Chunk  chunk     = null;
            long   bytesRead = 0;
            string key       = null;

            if (stream == null || !stream.CanRead || contentLength < 1)
            {
                return(false);
            }

            #endregion

            #region Single-Chunk

            if (contentLength <= _MinChunkSize)
            {
                byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
                key   = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
                chunk = new Chunk(
                    key,
                    contentLength,
                    0,
                    0,
                    chunkData);
                chunks.Add(chunk);
                return(processChunk(chunk));
            }

            #endregion

            #region Process-Sliding-Window

            Streams streamWindow  = new Streams(stream, contentLength, _MinChunkSize, _ShiftCount);
            byte[]  currChunk     = null;
            long    chunkPosition = 0;  // should only be set at the beginning of a new chunk

            while (true)
            {
                byte[] window = streamWindow.GetNextChunk(out long tempPosition, out byte[] newData, out bool finalChunk);
                if (window == null)
                {
                    return(true);
                }
                if (currChunk == null)
                {
                    chunkPosition = tempPosition;
                }

                if (currChunk == null)
                {
                    // starting a new chunk
                    currChunk = new byte[window.Length];
                    Buffer.BlockCopy(window, 0, currChunk, 0, window.Length);
                }
                else
                {
                    // append new data
                    currChunk = DedupeCommon.AppendBytes(currChunk, newData);
                }

                byte[] md5Hash = DedupeCommon.Md5(window);
                if (DedupeCommon.IsZeroBytes(md5Hash, _BoundaryCheckBytes)
                    ||
                    (currChunk.Length >= _MaxChunkSize))
                {
                    #region Chunk-Boundary

                    key   = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(currChunk));
                    chunk = new Chunk(
                        key,
                        currChunk.Length,
                        chunks.Count,
                        chunkPosition,
                        currChunk);

                    if (!processChunk(chunk))
                    {
                        return(false);
                    }
                    chunk.Value = null;
                    chunks.Add(chunk);

                    chunk     = null;
                    currChunk = null;

                    streamWindow.AdvanceToNewChunk();

                    #endregion
                }
                else
                {
                    // do nothing, continue;
                }

                if (finalChunk)
                {
                    #region Final-Chunk

                    if (currChunk != null)
                    {
                        key   = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(currChunk));
                        chunk = new Chunk(
                            key,
                            currChunk.Length,
                            chunks.Count,
                            chunkPosition,
                            currChunk);

                        if (!processChunk(chunk))
                        {
                            return(false);
                        }
                        chunk.Value = null;
                        chunks.Add(chunk);

                        chunk     = null;
                        currChunk = null;
                        break;
                    }

                    #endregion
                }
            }

            #endregion

            return(true);
        }