/// <summary> /// Opens the input stream. /// </summary> /// <returns></returns> /// <exception cref="System.ArgumentException">StartOffset is beyond the end of the input file;StartOffset</exception> private Stream OpenInputStream() { return(_frontEnd.ReadStream(_metadata.InputFilePath, _segmentMetadata.Offset, _segmentMetadata.Length, _metadata.IsDownload)); }
private async Task <int> ProcessRequestAsync(long offset, long lengthToDownload) { // for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use. // This also protects the user from unintentionally using the file after a failed download. var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath; using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { outputStream.Seek(offset, SeekOrigin.Begin); _token.ThrowIfCancellationRequested(); int attemptCount = 0; int partialDataAttempts = 0; bool downloadCompleted = false; long dataReceived = 0; bool modifyLengthAndOffset = false; while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount) { _token.ThrowIfCancellationRequested(); try { // in the case where we got less than the expected amount of data, // only download the rest of the data from the previous request // instead of a new full buffer. if (modifyLengthAndOffset) { lengthToDownload -= dataReceived; dataReceived = 0; Console.WriteLine("process offset {0} to {1} of file {2}, try {3}", offset, lengthToDownload, _segmentMetadata.Path, attemptCount); } using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, offset, lengthToDownload, _metadata.IsDownload)) { readStream.CopyTo(outputStream, (int)lengthToDownload); } var lengthReturned = outputStream.Position - offset; // if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data if (lengthReturned > lengthToDownload) { throw new UploadFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - offset, offset, _metadata.InputFilePath, DateTime.Now.ToString())); } // we need to validate how many bytes have actually been copied to the read stream if ((lengthReturned > 0) && (lengthReturned < lengthToDownload) && ((offset + lengthReturned) < (_segmentMetadata.Offset + _segmentMetadata.Length))) { Console.WriteLine("Downloaded UNEXPECTED Lenght {0} Expected: {1} of file {2} offsite is {3}", lengthReturned, lengthToDownload, _segmentMetadata.Path, offset); partialDataAttempts++; offset += lengthReturned; localOffset += lengthReturned; modifyLengthAndOffset = true; dataReceived += lengthReturned; ReportProgress(localOffset, false); // we will wait before the next iteration, since something went wrong and we did not receive enough data. // this could be a throttling issue or an issue with the service itself. Either way, waiting should help // reduce the liklihood of additional failures. if (partialDataAttempts >= MaxBufferDownloadAttemptCount) { throw new UploadFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeateded server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path)); } WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token); } else { downloadCompleted = true; offset += lengthToDownload; localOffset += lengthToDownload; ReportProgress(localOffset, false); } } catch (Exception ex) { // update counts and reset for internal attempts attemptCount++; partialDataAttempts = 0; Console.WriteLine("Throw exception, when expected to download: {0} of file {1} offsite is {2}, LimiteLength is:{3} modify offset lenght {4}", lengthToDownload, _segmentMetadata.Path, offset, _segmentMetadata.Offset + _segmentMetadata.Length, modifyLengthAndOffset); Console.WriteLine(ex.ToString()); //if we tried more than the number of times we were allowed to, give up and throw the exception if (attemptCount >= MaxBufferDownloadAttemptCount) { ReportProgress(localOffset, true); Console.WriteLine("Throw exception, when expected to download: {0} of file {1} offsite is {2}, LimiteLength is:{3} modify offset lenght {4}", lengthToDownload, _segmentMetadata.Path, offset, _segmentMetadata.Offset + _segmentMetadata.Length, modifyLengthAndOffset); throw ex; } else { WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token); // forcibly put the stream back to where it should be based on where we think we are in the download. outputStream.Seek(offset, SeekOrigin.Begin); } } } } return(1); }
/// <summary> /// Downloads the segment contents. /// </summary> private void DownloadSegmentContents() { // set the current offset in the stream we are reading to the offset // that this segment starts at. long curOffset = _segmentMetadata.Offset; // set the offset of the local file that we are creating to the beginning of the local stream. // this value will be used to ensure that we are always reporting the right progress and that, // in the event of faiure, we reset the local stream to the proper location. long localOffset = 0; // determine the number of requests made based on length of file divded by 32MB max size requests var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength); // set the length remaining to ensure that only the exact number of bytes is ultimately downloaded // for this segment. var lengthRemaining = _segmentMetadata.Length; using (var outputStream = new FileStream(_segmentMetadata.Path, FileMode.Create)) { for (int i = 0; i < numRequests; i++) { _token.ThrowIfCancellationRequested(); int attemptCount = 0; bool downloadCompleted = false; while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount) { _token.ThrowIfCancellationRequested(); attemptCount++; try { // test to make sure that the remaining length is larger than the max size, otherwise just download the remaining length long lengthToDownload = (long)BufferLength; if (lengthRemaining - lengthToDownload < 0) { lengthToDownload = lengthRemaining; } using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload)) { readStream.CopyTo(outputStream); } downloadCompleted = true; lengthRemaining -= lengthToDownload; curOffset += lengthToDownload; localOffset += lengthToDownload; ReportProgress(localOffset, false); } catch (Exception ex) { //if we tried more than the number of times we were allowed to, give up and throw the exception if (attemptCount >= MaxBufferDownloadAttemptCount) { ReportProgress(localOffset, true); throw ex; } else { WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token); // forcibly put the stream back to where it should be based on where we think we are in the download. outputStream.Seek(localOffset, SeekOrigin.Begin); } } } } } }
/// <summary> /// Downloads the segment contents. /// </summary> private void DownloadSegmentContents() { // set the current offset in the stream we are reading to the offset // that this segment starts at. long curOffset = _segmentMetadata.Offset; // set the offset of the local file that we are creating to the beginning of the local stream. // this value will be used to ensure that we are always reporting the right progress and that, // in the event of faiure, we reset the local stream to the proper location. long localOffset = 0; // determine the number of requests made based on length of file divded by 32MB max size requests var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength); // set the length remaining to ensure that only the exact number of bytes is ultimately downloaded // for this segment. var lengthRemaining = _segmentMetadata.Length; // for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use. // This also protects the user from unintentionally using the file after a failed download. var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath; using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { outputStream.Seek(curOffset, SeekOrigin.Begin); for (int i = 0; i < numRequests; i++) { _token.ThrowIfCancellationRequested(); int attemptCount = 0; int partialDataAttempts = 0; bool downloadCompleted = false; long dataReceived = 0; bool modifyLengthAndOffset = false; while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount) { _token.ThrowIfCancellationRequested(); try { long lengthToDownload = (long)BufferLength; // in the case where we got less than the expected amount of data, // only download the rest of the data from the previous request // instead of a new full buffer. if (modifyLengthAndOffset) { lengthToDownload -= dataReceived; } // test to make sure that the remaining length is larger than the max size, // otherwise just download the remaining length. if (lengthRemaining - lengthToDownload < 0) { lengthToDownload = lengthRemaining; } using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload)) { readStream.CopyTo(outputStream, (int)lengthToDownload); } var lengthReturned = outputStream.Position - curOffset; // if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data if (lengthReturned > lengthToDownload) { var ex = new TransferFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - curOffset, curOffset, _metadata.InputFilePath, DateTime.Now.ToString())); TracingHelper.LogError(ex); throw ex; } // we need to validate how many bytes have actually been copied to the read stream if (lengthReturned < lengthToDownload) { partialDataAttempts++; lengthRemaining -= lengthReturned; curOffset += lengthReturned; localOffset += lengthReturned; modifyLengthAndOffset = true; dataReceived += lengthReturned; ReportProgress(localOffset, false); // we will wait before the next iteration, since something went wrong and we did not receive enough data. // this could be a throttling issue or an issue with the service itself. Either way, waiting should help // reduce the liklihood of additional failures. if (partialDataAttempts >= MaxBufferDownloadAttemptCount) { var ex = new TransferFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path)); TracingHelper.LogError(ex); throw ex; } var waitTime = WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token); TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} returned: {1} bytes. Expected: {2} bytes. Attempt: {3}. Wait time in ms before retry: {4}", _metadata.InputFilePath, lengthReturned, lengthToDownload, partialDataAttempts, waitTime); } else { downloadCompleted = true; lengthRemaining -= lengthToDownload; curOffset += lengthToDownload; localOffset += lengthToDownload; ReportProgress(localOffset, false); } } catch (Exception ex) { // update counts and reset for internal attempts attemptCount++; partialDataAttempts = 0; //if we tried more than the number of times we were allowed to, give up and throw the exception if (attemptCount >= MaxBufferDownloadAttemptCount) { ReportProgress(localOffset, true); TracingHelper.LogError(ex); throw ex; } else { var waitTime = WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token); TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}", _metadata.InputFilePath, attemptCount, ex, waitTime); // forcibly put the stream back to where it should be based on where we think we are in the download. outputStream.Seek(curOffset, SeekOrigin.Begin); } } } } // full validation of the segment. if (outputStream.Position - _segmentMetadata.Offset != _segmentMetadata.Length) { var ex = new TransferFailedException(string.Format("Post-download stream segment verification failed for file {2}: target stream has a length of {0}, expected {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", outputStream.Position - _segmentMetadata.Offset, _segmentMetadata.Length, _segmentMetadata.Path)); TracingHelper.LogError(ex); throw ex; } } }