/// <summary>
/// Opens the input stream.
/// </summary>
/// <returns></returns>
/// <exception cref="System.ArgumentException">StartOffset is beyond the end of the input file;StartOffset</exception>
private Stream OpenInputStream()
{
return(_frontEnd.ReadStream(_metadata.InputFilePath, _segmentMetadata.Offset, _segmentMetadata.Length, _metadata.IsDownload));
}
private async Task <int> ProcessRequestAsync(long offset, long lengthToDownload)
{
// for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use.
// This also protects the user from unintentionally using the file after a failed download.
var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath;
using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
outputStream.Seek(offset, SeekOrigin.Begin);
_token.ThrowIfCancellationRequested();
int attemptCount = 0;
int partialDataAttempts = 0;
bool downloadCompleted = false;
long dataReceived = 0;
bool modifyLengthAndOffset = false;
while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount)
{
_token.ThrowIfCancellationRequested();
try
{
// in the case where we got less than the expected amount of data,
// only download the rest of the data from the previous request
// instead of a new full buffer.
if (modifyLengthAndOffset)
{
lengthToDownload -= dataReceived;
dataReceived = 0;
Console.WriteLine("process offset {0} to {1} of file {2}, try {3}", offset, lengthToDownload, _segmentMetadata.Path, attemptCount);
}
using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, offset, lengthToDownload, _metadata.IsDownload))
{
readStream.CopyTo(outputStream, (int)lengthToDownload);
}
var lengthReturned = outputStream.Position - offset;
// if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data
if (lengthReturned > lengthToDownload)
{
throw new UploadFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - offset, offset, _metadata.InputFilePath, DateTime.Now.ToString()));
}
// we need to validate how many bytes have actually been copied to the read stream
if ((lengthReturned > 0) && (lengthReturned < lengthToDownload) && ((offset + lengthReturned) < (_segmentMetadata.Offset + _segmentMetadata.Length)))
{
Console.WriteLine("Downloaded UNEXPECTED Lenght {0} Expected: {1} of file {2} offsite is {3}", lengthReturned, lengthToDownload, _segmentMetadata.Path, offset);
partialDataAttempts++;
offset += lengthReturned;
localOffset += lengthReturned;
modifyLengthAndOffset = true;
dataReceived += lengthReturned;
ReportProgress(localOffset, false);
// we will wait before the next iteration, since something went wrong and we did not receive enough data.
// this could be a throttling issue or an issue with the service itself. Either way, waiting should help
// reduce the liklihood of additional failures.
if (partialDataAttempts >= MaxBufferDownloadAttemptCount)
{
throw new UploadFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeateded server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path));
}
WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token);
}
else
{
downloadCompleted = true;
offset += lengthToDownload;
localOffset += lengthToDownload;
ReportProgress(localOffset, false);
}
}
catch (Exception ex)
{
// update counts and reset for internal attempts
attemptCount++;
partialDataAttempts = 0;
Console.WriteLine("Throw exception, when expected to download: {0} of file {1} offsite is {2}, LimiteLength is:{3} modify offset lenght {4}", lengthToDownload, _segmentMetadata.Path, offset, _segmentMetadata.Offset + _segmentMetadata.Length, modifyLengthAndOffset);
Console.WriteLine(ex.ToString());
//if we tried more than the number of times we were allowed to, give up and throw the exception
if (attemptCount >= MaxBufferDownloadAttemptCount)
{
ReportProgress(localOffset, true);
Console.WriteLine("Throw exception, when expected to download: {0} of file {1} offsite is {2}, LimiteLength is:{3} modify offset lenght {4}", lengthToDownload, _segmentMetadata.Path, offset, _segmentMetadata.Offset + _segmentMetadata.Length, modifyLengthAndOffset);
throw ex;
}
else
{
WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token);
// forcibly put the stream back to where it should be based on where we think we are in the download.
outputStream.Seek(offset, SeekOrigin.Begin);
}
}
}
}
return(1);
}
/// <summary>
/// Downloads the segment contents.
/// </summary>
private void DownloadSegmentContents()
{
// set the current offset in the stream we are reading to the offset
// that this segment starts at.
long curOffset = _segmentMetadata.Offset;
// set the offset of the local file that we are creating to the beginning of the local stream.
// this value will be used to ensure that we are always reporting the right progress and that,
// in the event of faiure, we reset the local stream to the proper location.
long localOffset = 0;
// determine the number of requests made based on length of file divded by 32MB max size requests
var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength);
// set the length remaining to ensure that only the exact number of bytes is ultimately downloaded
// for this segment.
var lengthRemaining = _segmentMetadata.Length;
using (var outputStream = new FileStream(_segmentMetadata.Path, FileMode.Create))
{
for (int i = 0; i < numRequests; i++)
{
_token.ThrowIfCancellationRequested();
int attemptCount = 0;
bool downloadCompleted = false;
while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount)
{
_token.ThrowIfCancellationRequested();
attemptCount++;
try
{
// test to make sure that the remaining length is larger than the max size, otherwise just download the remaining length
long lengthToDownload = (long)BufferLength;
if (lengthRemaining - lengthToDownload < 0)
{
lengthToDownload = lengthRemaining;
}
using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload))
{
readStream.CopyTo(outputStream);
}
downloadCompleted = true;
lengthRemaining -= lengthToDownload;
curOffset += lengthToDownload;
localOffset += lengthToDownload;
ReportProgress(localOffset, false);
}
catch (Exception ex)
{
//if we tried more than the number of times we were allowed to, give up and throw the exception
if (attemptCount >= MaxBufferDownloadAttemptCount)
{
ReportProgress(localOffset, true);
throw ex;
}
else
{
WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token);
// forcibly put the stream back to where it should be based on where we think we are in the download.
outputStream.Seek(localOffset, SeekOrigin.Begin);
}
}
}
}
}
}
/// <summary>
/// Downloads the segment contents.
/// </summary>
private void DownloadSegmentContents()
{
// set the current offset in the stream we are reading to the offset
// that this segment starts at.
long curOffset = _segmentMetadata.Offset;
// set the offset of the local file that we are creating to the beginning of the local stream.
// this value will be used to ensure that we are always reporting the right progress and that,
// in the event of faiure, we reset the local stream to the proper location.
long localOffset = 0;
// determine the number of requests made based on length of file divded by 32MB max size requests
var numRequests = Math.Ceiling(_segmentMetadata.Length / BufferLength);
// set the length remaining to ensure that only the exact number of bytes is ultimately downloaded
// for this segment.
var lengthRemaining = _segmentMetadata.Length;
// for multi-segment files we append "inprogress" to indicate that the file is not yet ready for use.
// This also protects the user from unintentionally using the file after a failed download.
var streamName = _metadata.SegmentCount > 1 ? string.Format("{0}.inprogress", _metadata.TargetStreamPath) : _metadata.TargetStreamPath;
using (var outputStream = new FileStream(streamName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
outputStream.Seek(curOffset, SeekOrigin.Begin);
for (int i = 0; i < numRequests; i++)
{
_token.ThrowIfCancellationRequested();
int attemptCount = 0;
int partialDataAttempts = 0;
bool downloadCompleted = false;
long dataReceived = 0;
bool modifyLengthAndOffset = false;
while (!downloadCompleted && attemptCount < MaxBufferDownloadAttemptCount)
{
_token.ThrowIfCancellationRequested();
try
{
long lengthToDownload = (long)BufferLength;
// in the case where we got less than the expected amount of data,
// only download the rest of the data from the previous request
// instead of a new full buffer.
if (modifyLengthAndOffset)
{
lengthToDownload -= dataReceived;
}
// test to make sure that the remaining length is larger than the max size,
// otherwise just download the remaining length.
if (lengthRemaining - lengthToDownload < 0)
{
lengthToDownload = lengthRemaining;
}
using (var readStream = _frontEnd.ReadStream(_metadata.InputFilePath, curOffset, lengthToDownload, _metadata.IsDownload))
{
readStream.CopyTo(outputStream, (int)lengthToDownload);
}
var lengthReturned = outputStream.Position - curOffset;
// if we got more data than we asked for something went wrong and we should retry, since we can't trust the extra data
if (lengthReturned > lengthToDownload)
{
var ex = new TransferFailedException(string.Format("{4}: Did not download the expected amount of data in the request. Expected: {0}. Actual: {1}. From offset: {2} in remote file: {3}", lengthToDownload, outputStream.Position - curOffset, curOffset, _metadata.InputFilePath, DateTime.Now.ToString()));
TracingHelper.LogError(ex);
throw ex;
}
// we need to validate how many bytes have actually been copied to the read stream
if (lengthReturned < lengthToDownload)
{
partialDataAttempts++;
lengthRemaining -= lengthReturned;
curOffset += lengthReturned;
localOffset += lengthReturned;
modifyLengthAndOffset = true;
dataReceived += lengthReturned;
ReportProgress(localOffset, false);
// we will wait before the next iteration, since something went wrong and we did not receive enough data.
// this could be a throttling issue or an issue with the service itself. Either way, waiting should help
// reduce the liklihood of additional failures.
if (partialDataAttempts >= MaxBufferDownloadAttemptCount)
{
var ex = new TransferFailedException(string.Format("Failed to retrieve the requested data after {0} attempts for file {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", MaxBufferDownloadAttemptCount, _segmentMetadata.Path));
TracingHelper.LogError(ex);
throw ex;
}
var waitTime = WaitForRetry(partialDataAttempts, this.UseBackOffRetryStrategy, _token);
TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} returned: {1} bytes. Expected: {2} bytes. Attempt: {3}. Wait time in ms before retry: {4}",
_metadata.InputFilePath,
lengthReturned,
lengthToDownload,
partialDataAttempts,
waitTime);
}
else
{
downloadCompleted = true;
lengthRemaining -= lengthToDownload;
curOffset += lengthToDownload;
localOffset += lengthToDownload;
ReportProgress(localOffset, false);
}
}
catch (Exception ex)
{
// update counts and reset for internal attempts
attemptCount++;
partialDataAttempts = 0;
//if we tried more than the number of times we were allowed to, give up and throw the exception
if (attemptCount >= MaxBufferDownloadAttemptCount)
{
ReportProgress(localOffset, true);
TracingHelper.LogError(ex);
throw ex;
}
else
{
var waitTime = WaitForRetry(attemptCount, this.UseBackOffRetryStrategy, _token);
TracingHelper.LogInfo("DownloadSegmentContents: ReadStream at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}",
_metadata.InputFilePath,
attemptCount,
ex,
waitTime);
// forcibly put the stream back to where it should be based on where we think we are in the download.
outputStream.Seek(curOffset, SeekOrigin.Begin);
}
}
}
}
// full validation of the segment.
if (outputStream.Position - _segmentMetadata.Offset != _segmentMetadata.Length)
{
var ex = new TransferFailedException(string.Format("Post-download stream segment verification failed for file {2}: target stream has a length of {0}, expected {1}. This usually indicates repeated server-side throttling due to exceeding account bandwidth.", outputStream.Position - _segmentMetadata.Offset, _segmentMetadata.Length, _segmentMetadata.Path));
TracingHelper.LogError(ex);
throw ex;
}
}
}
