Ejemplo n.º 1
0
        public Downloader(BufferManager bufferManager, 
                            ILargeFileDownloadParameters parameters,
                            ConcurrentQueue<ChunkedFilePart> writeQueue,
                            ConcurrentStack<int> readStack,
                            Func<int, bool> downloadThrottle, 
                            int expectedChunkTimeInSeconds,
                            FailureToken failureToken,
                            Action<string> logger = null,
                            CancellationToken? cancellation = null,                             
                            Func<ILargeFileDownloadParameters, 
                            ISimpleHttpGetByRangeClient> clientFactory = null)
        {
            SimulateTimedOut = false;
            NonRetryableError = false;
            HeartBeat = DateTime.Now;
            cancellation = (cancellation != null) ? cancellation.Value : CancellationToken.None;

            DownloadWorkerThread = new Thread((() =>
                                 {
                                     try
                                     {
                                     clientFactory = clientFactory ?? ((p) => new SimpleHttpGetByRangeClient(p.Uri, bufferManager, expectedChunkTimeInSeconds * 1000 ));

                                     logger = logger ?? ((s) => { });

                                     ISimpleHttpGetByRangeClient client = clientFactory(parameters);
                                     int currentChunk;
                                     readStack.TryPop(out currentChunk);
                                     int delayThrottle = 0;

                                     try
                                         {

                                             while (currentChunk >= 0 && !cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected) //-1 when we are done
                                             {
                                                 logger(string.Format("[{1}] downloading: {0}", currentChunk,parameters.Id));
                                                 SimpleHttpResponse response = null;
                                                  var part = new ChunkedFilePart();
                                                 part.FileOffset =  GetChunkStart(currentChunk, parameters.MaxChunkSize);
                                                 part.Length = GetChunkSizeForCurrentChunk(parameters.FileSize,
                                                                                           parameters.MaxChunkSize,
                                                                                           currentChunk);
                                                 try
                                                 {
                                                     response = client.Get(parameters.Uri, part.FileOffset, part.Length);
                                                 }
                                                 catch (Exception e)
                                                 {
                                                     logger(string.Format("[{0}] {1}", parameters.Id, (e.InnerException != null
                                                             ? e.InnerException.Message
                                                             : e.Message)));

                                                     ExecuteAndSquash(client.Dispose);
                                                     client = clientFactory(parameters);
                                                 }

                                                 if (response != null && response.WasSuccessful)
                                                 {
                                                     part.Chunk = currentChunk;
                                                     part.Content = response.Content;
                                                     writeQueue.Enqueue(part);

                                                     // reset the throttle when the part is finally successful
                                                     delayThrottle = 0;
                                                     logger(string.Format("[{1}] downloaded: {0}", currentChunk,parameters.Id));

                                                     HeartBeat = DateTime.Now;

                                                     if (!readStack.TryPop(out currentChunk))
                                                     {
                                                         currentChunk = -1;
                                                     }
                                                     while (downloadThrottle(currentChunk))
                                                     {
                                                         logger(string.Format("[{1}] throttling for chunk: {0}", currentChunk,parameters.Id));
                                                         if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
                                                         {
                                                             Thread.Sleep(500);
                                                         }
                                                     }
                                                 }
                                                 else if (response == null || response.IsStatusCodeRetryable)
                                                 {
                                                     int sleepSecs = Math.Min((int)Math.Pow(4.95, delayThrottle), 600);
                                                     logger(string.Format("[{2}] sleeping: {0}, {1}s", currentChunk, sleepSecs, parameters.Id));
                                                     if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
                                                     {
                                                         Thread.Sleep(sleepSecs * 1000); // 4s, 25s, 120s, 600s
                                                         delayThrottle++;
                                                     }
                                                 }
                                                 else
                                                 {
                                                     logger(String.Format("[{3}] parameters.Uri:{0}  part.FileOffset:{1} part.Length:{2}", parameters.Uri, part.FileOffset, part.Length, parameters.Id));
                                                     logger(string.Format("[{1}] ERROR!NonRetryableError! going to trigger download failure because got Response.StatusCode: {0}", response.StatusCode, parameters.Id));
                                                     NonRetryableError = true;
                                                     failureToken.TriggerFailure();
                                                     break;
                                                     //throw new SimpleHttpClientException(response);
                                                 }
                                             }
                                         }
                                         finally
                                         {
                                             if (currentChunk >= 0  /*&& !NonRetryableError*/)
                                             {
                                                 //put it back on the stack, if it's poison everyone else will die
                                                 readStack.Push(currentChunk);
                                             }
                                             if (client != null)
                                             {
                                                 ExecuteAndSquash(client.Dispose);
                                             }
                                         }
                                     logger(String.Format("[{1}] Thread {0} done", Thread.CurrentThread.ManagedThreadId, parameters.Id));
                                     }
                                     catch (ThreadAbortException exc)
                                     {
                                         Console.WriteLine(exc);
                                         Thread.Sleep(1000);
                                         throw;
                                     }
                                 }));
        }
Ejemplo n.º 2
0
        public Downloader(BufferManager bufferManager,
                          ILargeFileDownloadParameters parameters,
                          ConcurrentQueue <ChunkedFilePart> writeQueue,
                          ConcurrentStack <int> readStack,
                          Func <int, bool> downloadThrottle,
                          int expectedChunkTimeInSeconds,
                          FailureToken failureToken,
                          Action <string> logger         = null,
                          CancellationToken?cancellation = null,
                          Func <ILargeFileDownloadParameters,
                                ISimpleHttpGetByRangeClient> clientFactory = null)
        {
            SimulateTimedOut  = false;
            NonRetryableError = false;
            HeartBeat         = DateTime.Now;
            cancellation      = (cancellation != null) ? cancellation.Value : CancellationToken.None;

            DownloadWorkerThread = new Thread((() =>
            {
                try
                {
                    clientFactory = clientFactory ?? ((p) => new SimpleHttpGetByRangeClient(p.Uri, bufferManager, expectedChunkTimeInSeconds * 1000));

                    logger = logger ?? ((s) => { });

                    ISimpleHttpGetByRangeClient client = clientFactory(parameters);
                    int currentChunk;
                    readStack.TryPop(out currentChunk);
                    int delayThrottle = 0;

                    try
                    {
                        while (currentChunk >= 0 && !cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)                      //-1 when we are done
                        {
                            logger(string.Format("[{1}] downloading: {0}", currentChunk, parameters.Id));
                            SimpleHttpResponse response = null;
                            var part = new ChunkedFilePart();
                            part.FileOffset = GetChunkStart(currentChunk, parameters.MaxChunkSize);
                            part.Length = GetChunkSizeForCurrentChunk(parameters.FileSize,
                                                                      parameters.MaxChunkSize,
                                                                      currentChunk);
                            try
                            {
                                response = client.Get(parameters.Uri, part.FileOffset, part.Length);
                            }
                            catch (Exception e)
                            {
                                logger(string.Format("[{0}] {1}", parameters.Id, (e.InnerException != null
                                                             ? e.InnerException.Message
                                                             : e.Message)));

                                ExecuteAndSquash(client.Dispose);
                                client = clientFactory(parameters);
                            }

                            if (response != null && response.WasSuccessful)
                            {
                                part.Chunk = currentChunk;
                                part.Content = response.Content;
                                writeQueue.Enqueue(part);

                                // reset the throttle when the part is finally successful
                                delayThrottle = 0;
                                logger(string.Format("[{1}] downloaded: {0}", currentChunk, parameters.Id));

                                HeartBeat = DateTime.Now;

                                if (!readStack.TryPop(out currentChunk))
                                {
                                    currentChunk = -1;
                                }
                                while (downloadThrottle(currentChunk))
                                {
                                    logger(string.Format("[{1}] throttling for chunk: {0}", currentChunk, parameters.Id));
                                    if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
                                    {
                                        Thread.Sleep(500);
                                    }
                                }
                            }
                            else if (response == null || response.IsStatusCodeRetryable)
                            {
                                int sleepSecs = Math.Min((int)Math.Pow(4.95, delayThrottle), 600);
                                logger(string.Format("[{2}] sleeping: {0}, {1}s", currentChunk, sleepSecs, parameters.Id));
                                if (!cancellation.Value.IsCancellationRequested && !failureToken.FailureDetected)
                                {
                                    Thread.Sleep(sleepSecs * 1000);                      // 4s, 25s, 120s, 600s
                                    delayThrottle++;
                                }
                            }
                            else
                            {
                                logger(String.Format("[{3}] parameters.Uri:{0}  part.FileOffset:{1} part.Length:{2}", parameters.Uri, part.FileOffset, part.Length, parameters.Id));
                                logger(string.Format("[{1}] ERROR!NonRetryableError! going to trigger download failure because got Response.StatusCode: {0}", response.StatusCode, parameters.Id));
                                NonRetryableError = true;
                                failureToken.TriggerFailure();
                                break;
                                //throw new SimpleHttpClientException(response);
                            }
                        }
                    }
                    finally
                    {
                        if (currentChunk >= 0 /*&& !NonRetryableError*/)
                        {
                            //put it back on the stack, if it's poison everyone else will die
                            readStack.Push(currentChunk);
                        }
                        if (client != null)
                        {
                            ExecuteAndSquash(client.Dispose);
                        }
                    }
                    logger(String.Format("[{1}] Thread {0} done", Thread.CurrentThread.ManagedThreadId, parameters.Id));
                }
                catch (ThreadAbortException exc)
                {
                    Console.WriteLine(exc);
                    Thread.Sleep(1000);
                    throw;
                }
            }));
        }