예제 #1
0
        // http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
        protected void ReadChunked(Stream stream)
        {
            BeginReceiveStreamFragments();

            string contentLengthHeader    = GetFirstHeaderValue("Content-Length");
            bool   hasContentLengthHeader = !string.IsNullOrEmpty(contentLengthHeader);
            int    realLength             = 0;

            if (hasContentLengthHeader)
            {
                hasContentLengthHeader = int.TryParse(contentLengthHeader, out realLength);
            }

            if (HTTPManager.Logger.Level == Logger.Loglevels.All)
            {
                VerboseLogging(string.Format("ReadChunked - hasContentLengthHeader: {0}, contentLengthHeader: {1} realLength: {2:N0}", hasContentLengthHeader.ToString(), contentLengthHeader, realLength));
            }

            using (var output = new BufferPoolMemoryStream())
            {
                int chunkLength = ReadChunkLength(stream);

                if (HTTPManager.Logger.Level == Logger.Loglevels.All)
                {
                    VerboseLogging(string.Format("chunkLength: {0:N0}", chunkLength));
                }

                byte[] buffer = BufferPool.Get(Mathf.NextPowerOfTwo(chunkLength), true);

                int contentLength = 0;

                // Progress report:
                long Downloaded          = 0;
                long DownloadLength      = hasContentLengthHeader ? realLength : chunkLength;
                bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess
#if !BESTHTTP_DISABLE_CACHING
                                                                                           || this.IsFromCache
#endif
                                                                                           );

                if (sendProgressChanged)
                {
                    RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
                }

                string encoding =
#if !BESTHTTP_DISABLE_CACHING
                    IsFromCache ? null :
#endif
                    GetFirstHeaderValue("content-encoding");
                bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip";

                Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null;

                while (chunkLength != 0)
                {
                    if (this.baseRequest.IsCancellationRequested)
                    {
                        return;
                    }

                    // To avoid more GC garbage we use only one buffer, and resize only if the next chunk doesn't fit.
                    if (buffer.Length < chunkLength)
                    {
                        BufferPool.Resize(ref buffer, chunkLength, true);
                    }

                    int readBytes = 0;

                    // Fill up the buffer
                    do
                    {
                        int bytes = stream.Read(buffer, readBytes, chunkLength - readBytes);
                        if (bytes <= 0)
                        {
                            throw ExceptionHelper.ServerClosedTCPStream();
                        }

                        readBytes += bytes;

                        // Progress report:
                        // Placing reporting inside this cycle will report progress much more frequent
                        Downloaded += bytes;

                        if (sendProgressChanged)
                        {
                            RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
                        }
                    } while (readBytes < chunkLength);

                    if (baseRequest.UseStreaming)
                    {
                        if (gzipped)
                        {
                            var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true);
                            if (decompressed.Data != null)
                            {
                                FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
                            }
                        }
                        else
                        {
                            FeedStreamFragment(buffer, 0, readBytes);
                        }
                    }
                    else
                    {
                        output.Write(buffer, 0, readBytes);
                    }

                    // Every chunk data has a trailing CRLF
                    ReadTo(stream, LF);

                    contentLength += readBytes;

                    // read the next chunk's length
                    chunkLength = ReadChunkLength(stream);

                    if (HTTPManager.Logger.Level == Logger.Loglevels.All)
                    {
                        VerboseLogging(string.Format("chunkLength: {0:N0}", chunkLength));
                    }

                    if (!hasContentLengthHeader && sendProgressChanged)
                    {
                        RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
                    }
                }

                BufferPool.Release(buffer);

                if (baseRequest.UseStreaming)
                {
                    if (gzipped)
                    {
                        var decompressed = decompressor.Decompress(null, 0, 0, true, true);
                        if (decompressed.Data != null)
                        {
                            FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
                        }
                    }

                    FlushRemainingFragmentBuffer();
                }

                // Read the trailing headers or the CRLF
                ReadHeaders(stream);

                // HTTP servers sometimes use compression (gzip) or deflate methods to optimize transmission.
                // How both chunked and gzip encoding interact is dictated by the two-staged encoding of HTTP:
                //  first the content stream is encoded as (Content-Encoding: gzip), after which the resulting byte stream is encoded for transfer using another encoder (Transfer-Encoding: chunked).
                //  This means that in case both compression and chunked encoding are enabled, the chunk encoding itself is not compressed, and the data in each chunk should not be compressed individually.
                //  The remote endpoint can decode the incoming stream by first decoding it with the Transfer-Encoding, followed by the specified Content-Encoding.
                // It would be a better implementation when the chunk would be decododed on-the-fly. Becouse now the whole stream must be downloaded, and then decoded. It needs more memory.
                if (!baseRequest.UseStreaming)
                {
                    this.Data = DecodeStream(output);
                }

                if (decompressor != null)
                {
                    decompressor.Dispose();
                }
            }
        }
예제 #2
0
        // No transfer-encoding just raw bytes.
        internal void ReadRaw(Stream stream, long contentLength)
        {
            BeginReceiveStreamFragments();

            // Progress report:
            long downloaded          = 0;
            long downloadLength      = contentLength;
            bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess
#if !BESTHTTP_DISABLE_CACHING
                                                                                       || this.IsFromCache
#endif
                                                                                       );

            if (sendProgressChanged)
            {
                RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, downloaded, downloadLength));
            }

            if (HTTPManager.Logger.Level == Logger.Loglevels.All)
            {
                VerboseLogging(string.Format("ReadRaw - contentLength: {0:N0}", contentLength));
            }

            string encoding =
#if !BESTHTTP_DISABLE_CACHING
                IsFromCache ? null :
#endif
                GetFirstHeaderValue("content-encoding");
            bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip";

            Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null;

            if (!baseRequest.UseStreaming && contentLength > 2147483646)
            {
                throw new OverflowException("You have to use STREAMING to download files bigger than 2GB!");
            }

            using (var output = new BufferPoolMemoryStream(baseRequest.UseStreaming ? 0 : (int)contentLength))
            {
                // Because of the last parameter, buffer's size can be larger than the requested but there's no reason to use
                //  an exact sized one if there's an larger one available in the pool. Later we will use the whole buffer.
                byte[] buffer    = BufferPool.Get(Math.Max(baseRequest.StreamFragmentSize, MinBufferSize), true);
                int    readBytes = 0;

                while (contentLength > 0)
                {
                    if (this.baseRequest.IsCancellationRequested)
                    {
                        return;
                    }

                    readBytes = 0;

                    do
                    {
                        // tryToReadCount contain how much bytes we want to read in once. We try to read the buffer fully in once,
                        //  but with a limit of the remaining contentLength.
                        int tryToReadCount = (int)Math.Min(Math.Min(int.MaxValue, contentLength), buffer.Length - readBytes);

                        int bytes = stream.Read(buffer, readBytes, tryToReadCount);

                        if (bytes <= 0)
                        {
                            throw ExceptionHelper.ServerClosedTCPStream();
                        }

                        readBytes     += bytes;
                        contentLength -= bytes;

                        // Progress report:
                        if (sendProgressChanged)
                        {
                            downloaded += bytes;
                            RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, downloaded, downloadLength));
                        }
                    } while (readBytes < buffer.Length && contentLength > 0);

                    if (baseRequest.UseStreaming)
                    {
                        if (gzipped)
                        {
                            var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true);
                            if (decompressed.Data != null)
                            {
                                FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
                            }
                        }
                        else
                        {
                            FeedStreamFragment(buffer, 0, readBytes);
                        }
                    }
                    else
                    {
                        output.Write(buffer, 0, readBytes);
                    }
                }
                ;

                BufferPool.Release(buffer);

                if (baseRequest.UseStreaming)
                {
                    if (gzipped)
                    {
                        var decompressed = decompressor.Decompress(null, 0, 0, true, true);
                        if (decompressed.Data != null)
                        {
                            FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
                        }
                    }

                    FlushRemainingFragmentBuffer();
                }

                if (!baseRequest.UseStreaming)
                {
                    this.Data = DecodeStream(output);
                }
            }

            if (decompressor != null)
            {
                decompressor.Dispose();
            }
        }
예제 #3
0
        protected void ReadUnknownSize(Stream stream)
        {
            // Progress report:
            long Downloaded          = 0;
            long DownloadLength      = 0;
            bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess
#if !BESTHTTP_DISABLE_CACHING
                                                                                       || this.IsFromCache
#endif
                                                                                       );

            string encoding =
#if !BESTHTTP_DISABLE_CACHING
                IsFromCache ? null :
#endif
                GetFirstHeaderValue("content-encoding");
            bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip";

            Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null;

            using (var output = new BufferPoolMemoryStream())
            {
                byte[] buffer = BufferPool.Get(Math.Max(baseRequest.StreamFragmentSize, MinBufferSize), false);

                if (HTTPManager.Logger.Level == Logger.Loglevels.All)
                {
                    VerboseLogging(string.Format("ReadUnknownSize - buffer size: {0:N0}", buffer.Length));
                }

                int readBytes = 0;
                int bytes     = 0;
                do
                {
                    readBytes = 0;

                    do
                    {
                        if (this.baseRequest.IsCancellationRequested)
                        {
                            return;
                        }

                        bytes = 0;

#if !NETFX_CORE || UNITY_EDITOR
                        NetworkStream networkStream = stream as NetworkStream;
                        // If we have the good-old NetworkStream, than we can use the DataAvailable property. On WP8 platforms, these are omitted... :/
                        if (networkStream != null && baseRequest.EnableSafeReadOnUnknownContentLength)
                        {
                            for (int i = readBytes; i < buffer.Length && networkStream.DataAvailable; ++i)
                            {
                                int read = stream.ReadByte();
                                if (read >= 0)
                                {
                                    buffer[i] = (byte)read;
                                    bytes++;
                                }
                                else
                                {
                                    break;
                                }
                            }
                        }
                        else // This will be good anyway, but a little slower.
#endif
                        {
                            bytes = stream.Read(buffer, readBytes, buffer.Length - readBytes);
                        }

                        readBytes += bytes;

                        // Progress report:
                        Downloaded    += bytes;
                        DownloadLength = Downloaded;

                        if (sendProgressChanged)
                        {
                            RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
                        }
                    } while (readBytes < buffer.Length && bytes > 0);

                    if (baseRequest.UseStreaming)
                    {
                        if (gzipped)
                        {
                            var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true);
                            if (decompressed.Data != null)
                            {
                                FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
                            }
                        }
                        else
                        {
                            FeedStreamFragment(buffer, 0, readBytes);
                        }
                    }
                    else
                    {
                        output.Write(buffer, 0, readBytes);
                    }
                } while (bytes > 0);

                BufferPool.Release(buffer);

                if (baseRequest.UseStreaming)
                {
                    if (gzipped)
                    {
                        var decompressed = decompressor.Decompress(null, 0, 0, true, true);
                        if (decompressed.Data != null)
                        {
                            FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
                        }
                    }

                    FlushRemainingFragmentBuffer();
                }

                if (!baseRequest.UseStreaming)
                {
                    this.Data = DecodeStream(output);
                }
            }

            if (decompressor != null)
            {
                decompressor.Dispose();
            }
        }