protected SimpleHttpClientException(SimpleHttpResponse response,
SerializationInfo info,
StreamingContext context)
: base(info, context)
{
HttpResponse = response;
}
private void HandleResponse(SimpleHttpResponse response)
{
if (BytesCached > 0)
{
var newStream = new MemoryStream((int)BytesCached);
stream.CopyTo(newStream);
stream.Dispose();
stream = newStream;
}
else
{
stream.Position = 0;
}
new MemoryStream(response.Content).CopyTo(stream);
stream.SetLength(stream.Position); // the last buffer may be smaller than the current length!
stream.Position = 0;
}
internal void GetChunk(long start, long end, uint retryCount = 2)
{
using (var client = new SimpleHttpGetByRangeClient(new Uri(url)))
{
SimpleHttpResponse response = client.Get(start, end - start);
if (response.WasSuccessful)
{
HandleResponse(response);
}
else if (retryCount > 0)
{
//retry
GetChunk(start, end, --retryCount);
}
else
{
throw new HttpException(string.Format("Could not retrieve content from {0} between {1} and {2}",
url, start, end));
}
}
}
public SimpleHttpClientException(SimpleHttpResponse response, string message, Exception inner)
: base(message, inner)
{
HttpResponse = response;
}
//
// For guidelines regarding the creation of new exception types, see
// http://msdn.microsoft.com/library/default.asp?url=/library/en-us/cpgenref/html/cpconerrorraisinghandlingguidelines.asp
// and
// http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dncscol/html/csharp07192001.asp
//
public SimpleHttpClientException(SimpleHttpResponse response)
{
HttpResponse = response;
}
public SimpleHttpClientException(SimpleHttpResponse response, string message, Exception inner) : base(message, inner)
{
HttpResponse = response;
}
protected SimpleHttpClientException(SimpleHttpResponse response,
SerializationInfo info,
StreamingContext context) : base(info, context)
{
HttpResponse = response;
}
//
// For guidelines regarding the creation of new exception types, see
// http://msdn.microsoft.com/library/default.asp?url=/library/en-us/cpgenref/html/cpconerrorraisinghandlingguidelines.asp
// and
// http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dncscol/html/csharp07192001.asp
//
public SimpleHttpClientException(SimpleHttpResponse response)
{
HttpResponse = response;
}
public Downloader(BufferManager bufferManager,
ILargeFileDownloadParameters parameters,
ConcurrentQueue <ChunkedFilePart> writeQueue,
ConcurrentStack <int> readStack,
Func <int, bool> downloadThrottle,
int expectedChunkTimeInSeconds,
FailureToken failureToken,
Action <string> logger = null,
CancellationToken?cancellation = null,
Func <ILargeFileDownloadParameters,
ISimpleHttpGetByRangeClient> clientFactory = null)
{
SimulateTimedOut = false;
NonRetryableError = false;
HeartBeat = DateTime.Now;
cancellation = (cancellation != null) ? cancellation.Value : CancellationToken.None;
DownloadWorkerThread = new Thread((() =>
{
try
{
clientFactory = clientFactory ?? ((p) => new SimpleHttpGetByRangeClient(p.Uri, bufferManager, expectedChunkTimeInSeconds * 1000));
logger = logger ?? ((s) => { });
ISimpleHttpGetByRangeClient client = clientFactory(parameters);
int currentChunk;
readStack.TryPop(out currentChunk);
int delayThrottle = 0;
try
{
while (currentChunk >= 0 && !cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected) //-1 when we are done
{
logger(string.Format("[{1}] downloading: {0}", currentChunk, parameters.Id));
SimpleHttpResponse response = null;
var part = new ChunkedFilePart();
part.FileOffset = GetChunkStart(currentChunk, parameters.MaxChunkSize);
part.Length = GetChunkSizeForCurrentChunk(parameters.FileSize,
parameters.MaxChunkSize,
currentChunk);
try
{
response = client.Get(parameters.Uri, part.FileOffset, part.Length);
}
catch (Exception e)
{
logger(string.Format("[{0}] {1}", parameters.Id, (e.InnerException != null
? e.InnerException.Message
: e.Message)));
ExecuteAndSquash(client.Dispose);
client = clientFactory(parameters);
}
if (response != null && response.WasSuccessful)
{
part.Chunk = currentChunk;
part.Content = response.Content;
writeQueue.Enqueue(part);
// reset the throttle when the part is finally successful
delayThrottle = 0;
logger(string.Format("[{1}] downloaded: {0}", currentChunk, parameters.Id));
HeartBeat = DateTime.Now;
if (!readStack.TryPop(out currentChunk))
{
currentChunk = -1;
}
while (downloadThrottle(currentChunk))
{
logger(string.Format("[{1}] throttling for chunk: {0}", currentChunk, parameters.Id));
if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
{
Thread.Sleep(500);
}
}
}
else if (response == null || response.IsStatusCodeRetryable)
{
int sleepSecs = Math.Min((int)Math.Pow(4.95, delayThrottle), 600);
logger(string.Format("[{2}] sleeping: {0}, {1}s", currentChunk, sleepSecs, parameters.Id));
if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
{
Thread.Sleep(sleepSecs * 1000); // 4s, 25s, 120s, 600s
delayThrottle++;
}
}
else
{
logger(String.Format("[{3}] parameters.Uri:{0} part.FileOffset:{1} part.Length:{2}", parameters.Uri, part.FileOffset, part.Length, parameters.Id));
logger(string.Format("[{1}] ERROR!NonRetryableError! going to trigger download failure because got Response.StatusCode: {0}", response.StatusCode, parameters.Id));
NonRetryableError = true;
failureToken.TriggerFailure();
break;
//throw new SimpleHttpClientException(response);
}
}
}
finally
{
if (currentChunk >= 0 /*&& !NonRetryableError*/)
{
//put it back on the stack, if it's poison everyone else will die
readStack.Push(currentChunk);
}
if (client != null)
{
ExecuteAndSquash(client.Dispose);
}
}
logger(String.Format("[{1}] Thread {0} done", Thread.CurrentThread.ManagedThreadId, parameters.Id));
}
catch (ThreadAbortException exc)
{
Console.WriteLine(exc);
Thread.Sleep(1000);
throw;
}
}));
}
public SimpleHttpResponse ParseResult(Stream stream, long length)
{
var buffer = bufferManager.GetBuffer(BUFFER_SIZE);
int bytesread = stream.Read(buffer, 0, buffer.Length);
byte[] initialReadBytes = bufferManager.GetBuffer((uint)bytesread);
SimpleHttpResponse response;
try
{
if (bytesread < 10)
{
throw new IOException(INVALID_HEADER_LENGTH);
}
Buffer.BlockCopy(buffer, 0, initialReadBytes, 0, bytesread);
//some calculations to determine how much data we are getting;
int bodyIndex = initialReadBytes.IndexOf(BODY_INDICATOR);
int bodyStarts = bodyIndex + BODY_INDICATOR.Length;
int statusCode;
var headers = HttpParser.GetHttpHeaders(initialReadBytes, bodyIndex, out statusCode);
if (statusCode >= 200 && statusCode <= 300)
{
long contentLength = long.Parse(headers[HttpParser.HttpHeaders.ContentLength]);
var dest = bufferManager.GetBuffer((uint)contentLength);
using (var outputStream = new MemoryStream(dest))
{
int destPlace = initialReadBytes.Length - bodyStarts;
long totalContent = contentLength + bodyStarts;
long left = totalContent - bytesread;
outputStream.Write(initialReadBytes, bodyStarts, destPlace);
while (left > 0)
{
if (bytesread == 0)
{
throw new IOException(STREAM_CLOSED_ERROR);
}
// Trace.WriteLine(string.Format("reading buffer {0}", (int)(left < buffer.Length ? left : buffer.Length)));
bytesread = stream.Read(buffer, 0, (int)(left < buffer.Length ? left : buffer.Length));
outputStream.Write(buffer, 0, bytesread);
left -= bytesread;
}
response = new SimpleHttpResponse(statusCode, dest, headers);
}
}
else
{
response = new SimpleHttpResponse(statusCode, null, headers);
}
}
finally
{
bufferManager.FreeBuffer(buffer);
bufferManager.FreeBuffer(initialReadBytes);
}
return(response);
}
private void HandleResponse(SimpleHttpResponse response)
{
if (BytesCached > 0)
{
var newStream = new MemoryStream((int) BytesCached);
stream.CopyTo(newStream);
stream.Dispose();
stream = newStream;
}
else
{
stream.Position = 0;
}
new MemoryStream(response.Content).CopyTo(stream);
stream.SetLength(stream.Position); // the last buffer may be smaller than the current length!
stream.Position = 0;
}
public SimpleHttpResponse ParseResult(Stream stream, long length)
{
var buffer = bufferManager.GetBuffer(BUFFER_SIZE);
int bytesread = stream.Read(buffer, 0, buffer.Length);
byte[] initialReadBytes = bufferManager.GetBuffer((uint)bytesread);
SimpleHttpResponse response;
try
{
if (bytesread < 10)
throw new IOException(INVALID_HEADER_LENGTH);
Buffer.BlockCopy(buffer, 0, initialReadBytes, 0, bytesread);
//some calculations to determine how much data we are getting;
int bodyIndex = initialReadBytes.IndexOf(BODY_INDICATOR);
int bodyStarts = bodyIndex + BODY_INDICATOR.Length;
int statusCode;
var headers = HttpParser.GetHttpHeaders(initialReadBytes, bodyIndex, out statusCode);
if (statusCode >= 200 && statusCode <= 300)
{
long contentLength = long.Parse(headers[HttpParser.HttpHeaders.ContentLength]);
var dest = bufferManager.GetBuffer((uint)contentLength);
using (var outputStream = new MemoryStream(dest))
{
int destPlace = initialReadBytes.Length - bodyStarts;
long totalContent = contentLength + bodyStarts;
long left = totalContent - bytesread;
outputStream.Write(initialReadBytes, bodyStarts, destPlace);
while (left > 0)
{
if (bytesread == 0)
throw new IOException(STREAM_CLOSED_ERROR);
// Trace.WriteLine(string.Format("reading buffer {0}", (int)(left < buffer.Length ? left : buffer.Length)));
bytesread = stream.Read(buffer, 0, (int)(left < buffer.Length ? left : buffer.Length));
outputStream.Write(buffer, 0, bytesread);
left -= bytesread;
}
response = new SimpleHttpResponse(statusCode, dest, headers);
}
}
else
response = new SimpleHttpResponse(statusCode, null, headers);
}
finally
{
bufferManager.FreeBuffer(buffer);
bufferManager.FreeBuffer(initialReadBytes);
}
return response;
}
