public Downloader(BufferManager bufferManager,
ILargeFileDownloadParameters parameters,
ConcurrentQueue<ChunkedFilePart> writeQueue,
ConcurrentStack<int> readStack,
Func<int, bool> downloadThrottle,
int expectedChunkTimeInSeconds,
FailureToken failureToken,
Action<string> logger = null,
CancellationToken? cancellation = null,
Func<ILargeFileDownloadParameters,
ISimpleHttpGetByRangeClient> clientFactory = null)
{
SimulateTimedOut = false;
NonRetryableError = false;
HeartBeat = DateTime.Now;
cancellation = (cancellation != null) ? cancellation.Value : CancellationToken.None;
DownloadWorkerThread = new Thread((() =>
{
try
{
clientFactory = clientFactory ?? ((p) => new SimpleHttpGetByRangeClient(p.Uri, bufferManager, expectedChunkTimeInSeconds * 1000 ));
logger = logger ?? ((s) => { });
ISimpleHttpGetByRangeClient client = clientFactory(parameters);
int currentChunk;
readStack.TryPop(out currentChunk);
int delayThrottle = 0;
try
{
while (currentChunk >= 0 && !cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected) //-1 when we are done
{
logger(string.Format("[{1}] downloading: {0}", currentChunk,parameters.Id));
SimpleHttpResponse response = null;
var part = new ChunkedFilePart();
part.FileOffset = GetChunkStart(currentChunk, parameters.MaxChunkSize);
part.Length = GetChunkSizeForCurrentChunk(parameters.FileSize,
parameters.MaxChunkSize,
currentChunk);
try
{
response = client.Get(parameters.Uri, part.FileOffset, part.Length);
}
catch (Exception e)
{
logger(string.Format("[{0}] {1}", parameters.Id, (e.InnerException != null
? e.InnerException.Message
: e.Message)));
ExecuteAndSquash(client.Dispose);
client = clientFactory(parameters);
}
if (response != null && response.WasSuccessful)
{
part.Chunk = currentChunk;
part.Content = response.Content;
writeQueue.Enqueue(part);
// reset the throttle when the part is finally successful
delayThrottle = 0;
logger(string.Format("[{1}] downloaded: {0}", currentChunk,parameters.Id));
HeartBeat = DateTime.Now;
if (!readStack.TryPop(out currentChunk))
{
currentChunk = -1;
}
while (downloadThrottle(currentChunk))
{
logger(string.Format("[{1}] throttling for chunk: {0}", currentChunk,parameters.Id));
if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
{
Thread.Sleep(500);
}
}
}
else if (response == null || response.IsStatusCodeRetryable)
{
int sleepSecs = Math.Min((int)Math.Pow(4.95, delayThrottle), 600);
logger(string.Format("[{2}] sleeping: {0}, {1}s", currentChunk, sleepSecs, parameters.Id));
if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
{
Thread.Sleep(sleepSecs * 1000); // 4s, 25s, 120s, 600s
delayThrottle++;
}
}
else
{
logger(String.Format("[{3}] parameters.Uri:{0} part.FileOffset:{1} part.Length:{2}", parameters.Uri, part.FileOffset, part.Length, parameters.Id));
logger(string.Format("[{1}] ERROR!NonRetryableError! going to trigger download failure because got Response.StatusCode: {0}", response.StatusCode, parameters.Id));
NonRetryableError = true;
failureToken.TriggerFailure();
break;
//throw new SimpleHttpClientException(response);
}
}
}
finally
{
if (currentChunk >= 0 /*&& !NonRetryableError*/)
{
//put it back on the stack, if it's poison everyone else will die
readStack.Push(currentChunk);
}
if (client != null)
{
ExecuteAndSquash(client.Dispose);
}
}
logger(String.Format("[{1}] Thread {0} done", Thread.CurrentThread.ManagedThreadId, parameters.Id));
}
catch (ThreadAbortException exc)
{
Console.WriteLine(exc);
Thread.Sleep(1000);
throw;
}
}));
}
public Downloader(BufferManager bufferManager,
ILargeFileDownloadParameters parameters,
ConcurrentQueue <ChunkedFilePart> writeQueue,
ConcurrentStack <int> readStack,
Func <int, bool> downloadThrottle,
int expectedChunkTimeInSeconds,
FailureToken failureToken,
Action <string> logger = null,
CancellationToken?cancellation = null,
Func <ILargeFileDownloadParameters,
ISimpleHttpGetByRangeClient> clientFactory = null)
{
SimulateTimedOut = false;
NonRetryableError = false;
HeartBeat = DateTime.Now;
cancellation = (cancellation != null) ? cancellation.Value : CancellationToken.None;
DownloadWorkerThread = new Thread((() =>
{
try
{
clientFactory = clientFactory ?? ((p) => new SimpleHttpGetByRangeClient(p.Uri, bufferManager, expectedChunkTimeInSeconds * 1000));
logger = logger ?? ((s) => { });
ISimpleHttpGetByRangeClient client = clientFactory(parameters);
int currentChunk;
readStack.TryPop(out currentChunk);
int delayThrottle = 0;
try
{
while (currentChunk >= 0 && !cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected) //-1 when we are done
{
logger(string.Format("[{1}] downloading: {0}", currentChunk, parameters.Id));
SimpleHttpResponse response = null;
var part = new ChunkedFilePart();
part.FileOffset = GetChunkStart(currentChunk, parameters.MaxChunkSize);
part.Length = GetChunkSizeForCurrentChunk(parameters.FileSize,
parameters.MaxChunkSize,
currentChunk);
try
{
response = client.Get(parameters.Uri, part.FileOffset, part.Length);
}
catch (Exception e)
{
logger(string.Format("[{0}] {1}", parameters.Id, (e.InnerException != null
? e.InnerException.Message
: e.Message)));
ExecuteAndSquash(client.Dispose);
client = clientFactory(parameters);
}
if (response != null && response.WasSuccessful)
{
part.Chunk = currentChunk;
part.Content = response.Content;
writeQueue.Enqueue(part);
// reset the throttle when the part is finally successful
delayThrottle = 0;
logger(string.Format("[{1}] downloaded: {0}", currentChunk, parameters.Id));
HeartBeat = DateTime.Now;
if (!readStack.TryPop(out currentChunk))
{
currentChunk = -1;
}
while (downloadThrottle(currentChunk))
{
logger(string.Format("[{1}] throttling for chunk: {0}", currentChunk, parameters.Id));
if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
{
Thread.Sleep(500);
}
}
}
else if (response == null || response.IsStatusCodeRetryable)
{
int sleepSecs = Math.Min((int)Math.Pow(4.95, delayThrottle), 600);
logger(string.Format("[{2}] sleeping: {0}, {1}s", currentChunk, sleepSecs, parameters.Id));
if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
{
Thread.Sleep(sleepSecs * 1000); // 4s, 25s, 120s, 600s
delayThrottle++;
}
}
else
{
logger(String.Format("[{3}] parameters.Uri:{0} part.FileOffset:{1} part.Length:{2}", parameters.Uri, part.FileOffset, part.Length, parameters.Id));
logger(string.Format("[{1}] ERROR!NonRetryableError! going to trigger download failure because got Response.StatusCode: {0}", response.StatusCode, parameters.Id));
NonRetryableError = true;
failureToken.TriggerFailure();
break;
//throw new SimpleHttpClientException(response);
}
}
}
finally
{
if (currentChunk >= 0 /*&& !NonRetryableError*/)
{
//put it back on the stack, if it's poison everyone else will die
readStack.Push(currentChunk);
}
if (client != null)
{
ExecuteAndSquash(client.Dispose);
}
}
logger(String.Format("[{1}] Thread {0} done", Thread.CurrentThread.ManagedThreadId, parameters.Id));
}
catch (ThreadAbortException exc)
{
Console.WriteLine(exc);
Thread.Sleep(1000);
throw;
}
}));
}
