示例#1
0
        /// <summary>
        /// Read data from the stream into the specified buffer and increment the stream position.
        /// </summary>
        /// <param name="buffer">Byte array to use as a buffer.</param>
        /// <param name="offset">The offset within the buffer indicating where to copy the read data.</param>
        /// <param name="count">The number of bytes to populate within the buffer.</param>
        /// <returns>An integer representing the number of bytes read.</returns>
        public override int Read(byte[] buffer, int offset, int count)
        {
            if (buffer == null)
            {
                throw new ArgumentNullException(nameof(buffer));
            }
            if (offset < 0)
            {
                throw new ArgumentOutOfRangeException("Offset must be zero or greater.");
            }
            if (count < 0)
            {
                throw new ArgumentOutOfRangeException("Count must be zero or greater.");
            }
            if ((offset + count) > buffer.Length)
            {
                throw new ArgumentOutOfRangeException("Offset and count combined must not exceed buffer length.");
            }
            if (offset >= Length)
            {
                return(0);
            }

            DedupeObjectMap map = _Database.GetObjectMapForPosition(_Metadata.Key, _Position);

            if (map == null)
            {
                return(0);
            }

            if (map.ChunkAddress > Position)
            {
                throw new IOException("Data error while reading chunks from object.");
            }

            byte[] chunkData = _Callbacks.ReadChunk(map.ChunkKey);

            int chunkDataReadStart = 0;

            if (map.ChunkAddress < Position)
            {
                chunkDataReadStart += (int)(Position - map.ChunkAddress);
            }

            int bytesAvailInChunk = (int)(map.ChunkLength - chunkDataReadStart);

            if (count >= bytesAvailInChunk)
            {
                Buffer.BlockCopy(chunkData, chunkDataReadStart, buffer, offset, bytesAvailInChunk);
                _Position += bytesAvailInChunk;
                return(bytesAvailInChunk);
            }
            else
            {
                Buffer.BlockCopy(chunkData, chunkDataReadStart, buffer, offset, count);
                _Position += count;
                return(count);
            }
        }
示例#2
0
        private List <DedupeChunk> ChunkStream(string key, long contentLength, Stream stream, Action <DedupeChunk, DedupeObjectMap> processChunk)
        {
            if (String.IsNullOrEmpty(key))
            {
                throw new ArgumentNullException(nameof(key));
            }
            if (contentLength < 1)
            {
                throw new ArgumentException("Content length must be greater than zero.");
            }
            if (stream == null)
            {
                throw new ArgumentNullException(nameof(stream));
            }
            if (!stream.CanRead)
            {
                throw new ArgumentException("Cannot read from supplied stream.");
            }
            if (processChunk == null)
            {
                throw new ArgumentNullException(nameof(processChunk));
            }

            #region Initialize

            List <DedupeChunk> chunks = new List <DedupeChunk>();
            DedupeObjectMap    map    = null;
            DedupeChunk        chunk  = null;
            long   bytesRead          = 0;
            string chunkKey           = null;

            #endregion

            if (contentLength <= _Settings.MinChunkSize)
            {
                #region Single-Chunk

                byte[] chunkData = DedupeCommon.ReadBytesFromStream(stream, contentLength, out bytesRead);
                chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
                chunk    = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
                chunks.Add(chunk);

                map = new DedupeObjectMap(key, chunkKey, chunk.Length, 0, 0);
                processChunk(chunk, map);
                return(chunks);

                #endregion
            }
            else
            {
                #region Sliding-Window

                Streams streamWindow = new Streams(stream, contentLength, _Settings.MinChunkSize, _Settings.ShiftCount);
                byte[]  chunkData    = null;
                long    chunkAddress = 0;  // should only be set at the beginning of a new chunk

                while (true)
                {
                    byte[] newData    = null;
                    bool   finalChunk = false;

                    long   tempPosition = 0;
                    byte[] window       = streamWindow.GetNextChunk(out tempPosition, out newData, out finalChunk);
                    if (window == null)
                    {
                        return(chunks);
                    }
                    if (chunkData == null)
                    {
                        chunkAddress = tempPosition;
                    }

                    if (chunkData == null)
                    {
                        // starting a new chunk
                        chunkData = new byte[window.Length];
                        Buffer.BlockCopy(window, 0, chunkData, 0, window.Length);
                    }
                    else
                    {
                        // append new data
                        chunkData = DedupeCommon.AppendBytes(chunkData, newData);
                    }

                    byte[] md5Hash = DedupeCommon.Md5(window);
                    if (DedupeCommon.IsZeroBytes(md5Hash, _Settings.BoundaryCheckBytes) ||
                        chunkData.Length >= _Settings.MaxChunkSize)
                    {
                        #region Chunk-Boundary

                        chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));

                        chunk = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
                        map   = new DedupeObjectMap(key, chunk.Key, chunkData.Length, chunks.Count, chunkAddress);
                        processChunk(chunk, map);
                        chunk.Data = null;
                        chunks.Add(chunk);

                        chunk     = null;
                        chunkData = null;

                        streamWindow.AdvanceToNewChunk();

                        #endregion
                    }
                    else
                    {
                        // do nothing, continue;
                    }

                    if (finalChunk)
                    {
                        #region Final-Chunk

                        if (chunkData != null)
                        {
                            chunkKey = DedupeCommon.BytesToBase64(DedupeCommon.Sha256(chunkData));
                            chunk    = new DedupeChunk(chunkKey, chunkData.Length, 1, chunkData);
                            map      = new DedupeObjectMap(key, chunk.Key, chunk.Length, chunks.Count, chunkAddress);
                            processChunk(chunk, map);
                            chunk.Data = null;
                            chunks.Add(chunk);
                            break;
                        }

                        #endregion
                    }
                }

                #endregion
            }

            return(chunks);
        }