// http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
protected void ReadChunked(Stream stream)
{
BeginReceiveStreamFragments();
string contentLengthHeader = GetFirstHeaderValue("Content-Length");
bool hasContentLengthHeader = !string.IsNullOrEmpty(contentLengthHeader);
int realLength = 0;
if (hasContentLengthHeader)
{
hasContentLengthHeader = int.TryParse(contentLengthHeader, out realLength);
}
if (HTTPManager.Logger.Level == Logger.Loglevels.All)
{
VerboseLogging(string.Format("ReadChunked - hasContentLengthHeader: {0}, contentLengthHeader: {1} realLength: {2:N0}", hasContentLengthHeader.ToString(), contentLengthHeader, realLength));
}
using (var output = new BufferPoolMemoryStream())
{
int chunkLength = ReadChunkLength(stream);
if (HTTPManager.Logger.Level == Logger.Loglevels.All)
{
VerboseLogging(string.Format("chunkLength: {0:N0}", chunkLength));
}
byte[] buffer = BufferPool.Get(Mathf.NextPowerOfTwo(chunkLength), true);
int contentLength = 0;
// Progress report:
long Downloaded = 0;
long DownloadLength = hasContentLengthHeader ? realLength : chunkLength;
bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess
#if !BESTHTTP_DISABLE_CACHING
|| this.IsFromCache
#endif
);
if (sendProgressChanged)
{
RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
}
string encoding =
#if !BESTHTTP_DISABLE_CACHING
IsFromCache ? null :
#endif
GetFirstHeaderValue("content-encoding");
bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip";
Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null;
while (chunkLength != 0)
{
if (this.baseRequest.IsCancellationRequested)
{
return;
}
// To avoid more GC garbage we use only one buffer, and resize only if the next chunk doesn't fit.
if (buffer.Length < chunkLength)
{
BufferPool.Resize(ref buffer, chunkLength, true);
}
int readBytes = 0;
// Fill up the buffer
do
{
int bytes = stream.Read(buffer, readBytes, chunkLength - readBytes);
if (bytes <= 0)
{
throw ExceptionHelper.ServerClosedTCPStream();
}
readBytes += bytes;
// Progress report:
// Placing reporting inside this cycle will report progress much more frequent
Downloaded += bytes;
if (sendProgressChanged)
{
RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
}
} while (readBytes < chunkLength);
if (baseRequest.UseStreaming)
{
if (gzipped)
{
var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true);
if (decompressed.Data != null)
{
FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
}
}
else
{
FeedStreamFragment(buffer, 0, readBytes);
}
}
else
{
output.Write(buffer, 0, readBytes);
}
// Every chunk data has a trailing CRLF
ReadTo(stream, LF);
contentLength += readBytes;
// read the next chunk's length
chunkLength = ReadChunkLength(stream);
if (HTTPManager.Logger.Level == Logger.Loglevels.All)
{
VerboseLogging(string.Format("chunkLength: {0:N0}", chunkLength));
}
if (!hasContentLengthHeader && sendProgressChanged)
{
RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
}
}
BufferPool.Release(buffer);
if (baseRequest.UseStreaming)
{
if (gzipped)
{
var decompressed = decompressor.Decompress(null, 0, 0, true, true);
if (decompressed.Data != null)
{
FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
}
}
FlushRemainingFragmentBuffer();
}
// Read the trailing headers or the CRLF
ReadHeaders(stream);
// HTTP servers sometimes use compression (gzip) or deflate methods to optimize transmission.
// How both chunked and gzip encoding interact is dictated by the two-staged encoding of HTTP:
// first the content stream is encoded as (Content-Encoding: gzip), after which the resulting byte stream is encoded for transfer using another encoder (Transfer-Encoding: chunked).
// This means that in case both compression and chunked encoding are enabled, the chunk encoding itself is not compressed, and the data in each chunk should not be compressed individually.
// The remote endpoint can decode the incoming stream by first decoding it with the Transfer-Encoding, followed by the specified Content-Encoding.
// It would be a better implementation when the chunk would be decododed on-the-fly. Becouse now the whole stream must be downloaded, and then decoded. It needs more memory.
if (!baseRequest.UseStreaming)
{
this.Data = DecodeStream(output);
}
if (decompressor != null)
{
decompressor.Dispose();
}
}
}
// No transfer-encoding just raw bytes.
internal void ReadRaw(Stream stream, long contentLength)
{
BeginReceiveStreamFragments();
// Progress report:
long downloaded = 0;
long downloadLength = contentLength;
bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess
#if !BESTHTTP_DISABLE_CACHING
|| this.IsFromCache
#endif
);
if (sendProgressChanged)
{
RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, downloaded, downloadLength));
}
if (HTTPManager.Logger.Level == Logger.Loglevels.All)
{
VerboseLogging(string.Format("ReadRaw - contentLength: {0:N0}", contentLength));
}
string encoding =
#if !BESTHTTP_DISABLE_CACHING
IsFromCache ? null :
#endif
GetFirstHeaderValue("content-encoding");
bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip";
Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null;
if (!baseRequest.UseStreaming && contentLength > 2147483646)
{
throw new OverflowException("You have to use STREAMING to download files bigger than 2GB!");
}
using (var output = new BufferPoolMemoryStream(baseRequest.UseStreaming ? 0 : (int)contentLength))
{
// Because of the last parameter, buffer's size can be larger than the requested but there's no reason to use
// an exact sized one if there's an larger one available in the pool. Later we will use the whole buffer.
byte[] buffer = BufferPool.Get(Math.Max(baseRequest.StreamFragmentSize, MinBufferSize), true);
int readBytes = 0;
while (contentLength > 0)
{
if (this.baseRequest.IsCancellationRequested)
{
return;
}
readBytes = 0;
do
{
// tryToReadCount contain how much bytes we want to read in once. We try to read the buffer fully in once,
// but with a limit of the remaining contentLength.
int tryToReadCount = (int)Math.Min(Math.Min(int.MaxValue, contentLength), buffer.Length - readBytes);
int bytes = stream.Read(buffer, readBytes, tryToReadCount);
if (bytes <= 0)
{
throw ExceptionHelper.ServerClosedTCPStream();
}
readBytes += bytes;
contentLength -= bytes;
// Progress report:
if (sendProgressChanged)
{
downloaded += bytes;
RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, downloaded, downloadLength));
}
} while (readBytes < buffer.Length && contentLength > 0);
if (baseRequest.UseStreaming)
{
if (gzipped)
{
var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true);
if (decompressed.Data != null)
{
FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
}
}
else
{
FeedStreamFragment(buffer, 0, readBytes);
}
}
else
{
output.Write(buffer, 0, readBytes);
}
}
;
BufferPool.Release(buffer);
if (baseRequest.UseStreaming)
{
if (gzipped)
{
var decompressed = decompressor.Decompress(null, 0, 0, true, true);
if (decompressed.Data != null)
{
FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
}
}
FlushRemainingFragmentBuffer();
}
if (!baseRequest.UseStreaming)
{
this.Data = DecodeStream(output);
}
}
if (decompressor != null)
{
decompressor.Dispose();
}
}
protected void ReadUnknownSize(Stream stream)
{
// Progress report:
long Downloaded = 0;
long DownloadLength = 0;
bool sendProgressChanged = this.baseRequest.OnDownloadProgress != null && (this.IsSuccess
#if !BESTHTTP_DISABLE_CACHING
|| this.IsFromCache
#endif
);
string encoding =
#if !BESTHTTP_DISABLE_CACHING
IsFromCache ? null :
#endif
GetFirstHeaderValue("content-encoding");
bool gzipped = !string.IsNullOrEmpty(encoding) && encoding == "gzip";
Decompression.GZipDecompressor decompressor = gzipped ? new Decompression.GZipDecompressor(256) : null;
using (var output = new BufferPoolMemoryStream())
{
byte[] buffer = BufferPool.Get(Math.Max(baseRequest.StreamFragmentSize, MinBufferSize), false);
if (HTTPManager.Logger.Level == Logger.Loglevels.All)
{
VerboseLogging(string.Format("ReadUnknownSize - buffer size: {0:N0}", buffer.Length));
}
int readBytes = 0;
int bytes = 0;
do
{
readBytes = 0;
do
{
if (this.baseRequest.IsCancellationRequested)
{
return;
}
bytes = 0;
#if !NETFX_CORE || UNITY_EDITOR
NetworkStream networkStream = stream as NetworkStream;
// If we have the good-old NetworkStream, than we can use the DataAvailable property. On WP8 platforms, these are omitted... :/
if (networkStream != null && baseRequest.EnableSafeReadOnUnknownContentLength)
{
for (int i = readBytes; i < buffer.Length && networkStream.DataAvailable; ++i)
{
int read = stream.ReadByte();
if (read >= 0)
{
buffer[i] = (byte)read;
bytes++;
}
else
{
break;
}
}
}
else // This will be good anyway, but a little slower.
#endif
{
bytes = stream.Read(buffer, readBytes, buffer.Length - readBytes);
}
readBytes += bytes;
// Progress report:
Downloaded += bytes;
DownloadLength = Downloaded;
if (sendProgressChanged)
{
RequestEventHelper.EnqueueRequestEvent(new RequestEventInfo(this.baseRequest, RequestEvents.DownloadProgress, Downloaded, DownloadLength));
}
} while (readBytes < buffer.Length && bytes > 0);
if (baseRequest.UseStreaming)
{
if (gzipped)
{
var decompressed = decompressor.Decompress(buffer, 0, readBytes, false, true);
if (decompressed.Data != null)
{
FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
}
}
else
{
FeedStreamFragment(buffer, 0, readBytes);
}
}
else
{
output.Write(buffer, 0, readBytes);
}
} while (bytes > 0);
BufferPool.Release(buffer);
if (baseRequest.UseStreaming)
{
if (gzipped)
{
var decompressed = decompressor.Decompress(null, 0, 0, true, true);
if (decompressed.Data != null)
{
FeedStreamFragment(decompressed.Data, 0, decompressed.Length);
}
}
FlushRemainingFragmentBuffer();
}
if (!baseRequest.UseStreaming)
{
this.Data = DecodeStream(output);
}
}
if (decompressor != null)
{
decompressor.Dispose();
}
}