Example #1
0
 /// <summary>
 /// Opens the input stream.
 /// </summary>
 /// <returns></returns>
 /// <exception cref="System.ArgumentException">StartOffset is beyond the end of the input file;StartOffset</exception>
 private Stream OpenInputStream()
 {
     return(_frontEnd.ReadStream(_metadata.InputFilePath, _segmentMetadata.Offset, _segmentMetadata.Length, _metadata.IsDownload));
 }
Example #2
0
        private async Task <int> ProcessRequestAsync(long offset, long lengthToDownload)
        {
            // for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use.
            // This also protects the user from unintentionally using the file after a failed download.
            var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath;

            using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
            {
                outputStream.Seek(offset, SeekOrigin.Begin);

                _token.ThrowIfCancellationRequested();
                int  attemptCount          = 0;
                int  partialDataAttempts   = 0;
                bool downloadCompleted     = false;
                long dataReceived          = 0;
                bool modifyLengthAndOffset = false;


                while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount)
                {
                    _token.ThrowIfCancellationRequested();
                    try
                    {
                        // in the case where we got less than the expected amount of data,
                        // only download the rest of the data from the previous request
                        // instead of a new full buffer.
                        if (modifyLengthAndOffset)
                        {
                            lengthToDownload -= dataReceived;
                            dataReceived      = 0;
                            Console.WriteLine("process offset {0} to {1} of file {2}, try {3}", offset, lengthToDownload, _segmentMetadata.Path, attemptCount);
                        }


                        using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, offset, lengthToDownload, _metadata.IsDownload))
                        {
                            readStream.CopyTo(outputStream, (int)lengthToDownload);
                        }

                        var lengthReturned = outputStream.Position - offset;

                        // if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data
                        if (lengthReturned > lengthToDownload)
                        {
                            throw new UploadFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - offset, offset, _metadata.InputFilePath, DateTime.Now.ToString()));
                        }

                        // we need to validate how many bytes have actually been copied to the read stream
                        if ((lengthReturned > 0) && (lengthReturned < lengthToDownload) && ((offset + lengthReturned) < (_segmentMetadata.Offset + _segmentMetadata.Length)))
                        {
                            Console.WriteLine("Downloaded UNEXPECTED Lenght {0} Expected: {1} of file {2} offsite is {3}", lengthReturned, lengthToDownload, _segmentMetadata.Path, offset);
                            partialDataAttempts++;
                            offset               += lengthReturned;
                            localOffset          += lengthReturned;
                            modifyLengthAndOffset = true;
                            dataReceived         += lengthReturned;
                            ReportProgress(localOffset, false);


                            // we will wait before the next iteration, since something went wrong and we did not receive enough data.
                            // this could be a throttling issue or an issue with the service itself. Either way, waiting should help
                            // reduce the liklihood of additional failures.
                            if (partialDataAttempts >= MaxBufferDownloadAttemptCount)
                            {
                                throw new UploadFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeateded server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path));
                            }

                            WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token);
                        }
                        else
                        {
                            downloadCompleted = true;
                            offset           += lengthToDownload;
                            localOffset      += lengthToDownload;
                            ReportProgress(localOffset, false);
                        }
                    }
                    catch (Exception ex)
                    {
                        // update counts and reset for internal attempts
                        attemptCount++;
                        partialDataAttempts = 0;
                        Console.WriteLine("Throw exception, when expected to download: {0} of file {1} offsite is {2}, LimiteLength is:{3} modify offset lenght {4}", lengthToDownload, _segmentMetadata.Path, offset, _segmentMetadata.Offset + _segmentMetadata.Length, modifyLengthAndOffset);
                        Console.WriteLine(ex.ToString());
                        //if we tried more than the number of times we were allowed to, give up and throw the exception
                        if (attemptCount >= MaxBufferDownloadAttemptCount)
                        {
                            ReportProgress(localOffset, true);
                            Console.WriteLine("Throw exception, when expected to download: {0} of file {1} offsite is {2}, LimiteLength is:{3} modify offset lenght {4}", lengthToDownload, _segmentMetadata.Path, offset, _segmentMetadata.Offset + _segmentMetadata.Length, modifyLengthAndOffset);
                            throw ex;
                        }
                        else
                        {
                            WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token);

                            // forcibly put the stream back to where it should be based on where we think we are in the download.
                            outputStream.Seek(offset, SeekOrigin.Begin);
                        }
                    }
                }
            }
            return(1);
        }
        /// <summary>
        /// Downloads the segment contents.
        /// </summary>
        private void DownloadSegmentContents()
        {
            // set the current offset in the stream we are reading to the offset
            // that this segment starts at.
            long curOffset = _segmentMetadata.Offset;

            // set the offset of the local file that we are creating to the beginning of the local stream.
            // this value will be used to ensure that we are always reporting the right progress and that,
            // in the event of faiure, we reset the local stream to the proper location.
            long localOffset = 0;

            // determine the number of requests made based on length of file divded by 32MB max size requests
            var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength);
            // set the length remaining to ensure that only the exact number of bytes is ultimately downloaded
            // for this segment.
            var lengthRemaining = _segmentMetadata.Length;
            using (var outputStream = new FileStream(_segmentMetadata.Path, FileMode.Create))
            {
                for (int i = 0; i < numRequests; i++)
                {
                    _token.ThrowIfCancellationRequested();
                    int attemptCount = 0;
                    bool downloadCompleted = false;
                    while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount)
                    {
                        _token.ThrowIfCancellationRequested();
                        attemptCount++;
                        try
                        {
                            // test to make sure that the remaining length is larger than the max size, otherwise just download the remaining length
                            long lengthToDownload = (long)BufferLength;
                            if (lengthRemaining - lengthToDownload < 0)
                            {
                                lengthToDownload = lengthRemaining;
                            }

                            using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload))
                            {
                                readStream.CopyTo(outputStream);
                            }

                            downloadCompleted = true;
                            lengthRemaining -= lengthToDownload;
                            curOffset += lengthToDownload;
                            localOffset += lengthToDownload;
                            ReportProgress(localOffset, false);
                        }
                        catch (Exception ex)
                        {
                            //if we tried more than the number of times we were allowed to, give up and throw the exception
                            if (attemptCount >= MaxBufferDownloadAttemptCount)
                            {
                                ReportProgress(localOffset, true);
                                throw ex;
                            }
                            else
                            {
                                WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token);
                                
                                // forcibly put the stream back to where it should be based on where we think we are in the download.
                                outputStream.Seek(localOffset, SeekOrigin.Begin); 
                            }
                        }
                    }
                }
            }
        }
Example #4
0
        /// <summary>
        /// Downloads the segment contents.
        /// </summary>
        private void DownloadSegmentContents()
        {
            // set the current offset in the stream we are reading to the offset
            // that this segment starts at.
            long curOffset = _segmentMetadata.Offset;

            // set the offset of the local file that we are creating to the beginning of the local stream.
            // this value will be used to ensure that we are always reporting the right progress and that,
            // in the event of faiure, we reset the local stream to the proper location.
            long localOffset = 0;

            // determine the number of requests made based on length of file divded by 32MB max size requests
            var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength);
            // set the length remaining to ensure that only the exact number of bytes is ultimately downloaded
            // for this segment.
            var lengthRemaining = _segmentMetadata.Length;

            // for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use.
            // This also protects the user from unintentionally using the file after a failed download.
            var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath;

            using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
            {
                outputStream.Seek(curOffset, SeekOrigin.Begin);
                for (int i = 0; i < numRequests; i++)
                {
                    _token.ThrowIfCancellationRequested();
                    int  attemptCount          = 0;
                    int  partialDataAttempts   = 0;
                    bool downloadCompleted     = false;
                    long dataReceived          = 0;
                    bool modifyLengthAndOffset = false;
                    while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount)
                    {
                        _token.ThrowIfCancellationRequested();
                        try
                        {
                            long lengthToDownload = (long)BufferLength;

                            // in the case where we got less than the expected amount of data,
                            // only download the rest of the data from the previous request
                            // instead of a new full buffer.
                            if (modifyLengthAndOffset)
                            {
                                lengthToDownload -= dataReceived;
                            }

                            // test to make sure that the remaining length is larger than the max size,
                            // otherwise just download the remaining length.
                            if (lengthRemaining - lengthToDownload < 0)
                            {
                                lengthToDownload = lengthRemaining;
                            }

                            using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload))
                            {
                                readStream.CopyTo(outputStream, (int)lengthToDownload);
                            }

                            var lengthReturned = outputStream.Position - curOffset;

                            // if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data
                            if (lengthReturned > lengthToDownload)
                            {
                                var ex = new TransferFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - curOffset, curOffset, _metadata.InputFilePath, DateTime.Now.ToString()));
                                TracingHelper.LogError(ex);

                                throw ex;
                            }

                            // we need to validate how many bytes have actually been copied to the read stream
                            if (lengthReturned < lengthToDownload)
                            {
                                partialDataAttempts++;
                                lengthRemaining      -= lengthReturned;
                                curOffset            += lengthReturned;
                                localOffset          += lengthReturned;
                                modifyLengthAndOffset = true;
                                dataReceived         += lengthReturned;
                                ReportProgress(localOffset, false);

                                // we will wait before the next iteration, since something went wrong and we did not receive enough data.
                                // this could be a throttling issue or an issue with the service itself. Either way, waiting should help
                                // reduce the liklihood of additional failures.
                                if (partialDataAttempts >= MaxBufferDownloadAttemptCount)
                                {
                                    var ex = new TransferFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path));
                                    TracingHelper.LogError(ex);

                                    throw ex;
                                }

                                var waitTime = WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token);
                                TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} returned: {1} bytes. Expected: {2} bytes. Attempt: {3}. Wait time in ms before retry: {4}",
                                                      _metadata.InputFilePath,
                                                      lengthReturned,
                                                      lengthToDownload,
                                                      partialDataAttempts,
                                                      waitTime);
                            }
                            else
                            {
                                downloadCompleted = true;
                                lengthRemaining  -= lengthToDownload;
                                curOffset        += lengthToDownload;
                                localOffset      += lengthToDownload;
                                ReportProgress(localOffset, false);
                            }
                        }
                        catch (Exception ex)
                        {
                            // update counts and reset for internal attempts
                            attemptCount++;
                            partialDataAttempts = 0;

                            //if we tried more than the number of times we were allowed to, give up and throw the exception
                            if (attemptCount >= MaxBufferDownloadAttemptCount)
                            {
                                ReportProgress(localOffset, true);
                                TracingHelper.LogError(ex);

                                throw ex;
                            }
                            else
                            {
                                var waitTime = WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token);
                                TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}",
                                                      _metadata.InputFilePath,
                                                      attemptCount,
                                                      ex,
                                                      waitTime);

                                // forcibly put the stream back to where it should be based on where we think we are in the download.
                                outputStream.Seek(curOffset, SeekOrigin.Begin);
                            }
                        }
                    }
                }

                // full validation of the segment.
                if (outputStream.Position - _segmentMetadata.Offset != _segmentMetadata.Length)
                {
                    var ex = new TransferFailedException(string.Format("Post-download stream segment verification failed for file {2}: target stream has a length of {0}, expected {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", outputStream.Position - _segmentMetadata.Offset, _segmentMetadata.Length, _segmentMetadata.Path));
                    TracingHelper.LogError(ex);

                    throw ex;
                }
            }
        }