/// <summary>
/// Verifies the uploaded stream.
/// </summary>
/// <exception cref="UploadFailedException"></exception>
private void VerifyUploadedStream()
{
//verify that the remote stream has the length we expected.
var retryCount = 0;
long remoteLength = -1;
while (retryCount < MaxBufferUploadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
remoteLength = _frontEnd.GetStreamLength(_segmentMetadata.Path);
break;
}
catch (Exception)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= MaxBufferUploadAttemptCount)
{
throw;
}
WaitForRetry(retryCount, this.UseBackOffRetryStrategy, _token);
}
}
if (_segmentMetadata.Length != remoteLength)
{
throw new UploadFailedException(string.Format("Post-upload stream verification failed: target stream has a length of {0}, expected {1}", remoteLength, _segmentMetadata.Length));
}
}
/// <summary>
/// Constructs a new TransferMetadata from the given parameters.
/// </summary>
/// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param>
/// <param name="transferParameters">The parameters to use for constructing this metadata.</param>
/// <param name="frontEnd">The front end. This is used only in the constructor for determining file length</param>
internal TransferMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontEnd, long fileSize = -1)
{
this.MetadataFilePath = metadataFilePath;
this.TransferId = Guid.NewGuid().ToString("N");
this.InputFilePath = transferParameters.InputFilePath;
this.TargetStreamPath = transferParameters.TargetStreamPath;
this.IsDownload = transferParameters.IsDownload;
this.SegmentStreamDirectory = GetSegmentStreamDirectory();
this.IsBinary = transferParameters.IsBinary;
this.FileLength = fileSize < 0 ? frontEnd.GetStreamLength(transferParameters.InputFilePath, !IsDownload) : fileSize;
this.EncodingCodePage = transferParameters.FileEncoding.CodePage;
// we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic.
// this protects us against agressive increase of thread count resulting in far more segments than
// is reasonable for a given file size. We also ensure that each segment is at least 256mb in size.
// This is the size that ensures we have the optimal storage creation in the store.
var preliminarySegmentCount = (int)Math.Ceiling((double)this.FileLength / transferParameters.MaxSegementLength);
this.SegmentCount = Math.Min(preliminarySegmentCount, TransferSegmentMetadata.CalculateSegmentCount(this.FileLength));
this.SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(this.FileLength, this.SegmentCount);
this.Segments = new TransferSegmentMetadata[this.SegmentCount];
for (int i = 0; i < this.SegmentCount; i++)
{
this.Segments[i] = new TransferSegmentMetadata(i, this);
}
if (!transferParameters.IsBinary && this.SegmentCount > 1 && !this.IsDownload)
{
this.AlignSegmentsToRecordBoundaries();
// ensure that nothing strange happened during alignment
this.ValidateConsistency();
}
// initialize the status to pending, since it is not yet done.
this.Status = SegmentTransferStatus.Pending;
}
/// <summary>
/// Verifies the downloaded stream.
/// </summary>
/// <exception cref="TransferFailedException"></exception>
internal void VerifyDownloadedStream()
{
//verify that the remote stream has the length we expected.
var retryCount = 0;
long remoteLength = -1;
while (retryCount < MaxBufferDownloadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
remoteLength = _frontEnd.GetStreamLength(_segmentMetadata.Path, _metadata.IsDownload);
break;
}
catch (Exception e)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= MaxBufferDownloadAttemptCount)
{
TracingHelper.LogError(e);
throw e;
}
var waitTime = WaitForRetry(retryCount, this.UseBackOffRetryStrategy, _token);
TracingHelper.LogInfo("VerifyDownloadedStream: GetStreamLength at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}",
_segmentMetadata.Path,
retryCount,
e,
waitTime);
}
}
if (_segmentMetadata.Length != remoteLength)
{
var ex = new TransferFailedException(string.Format("Post-download stream verification failed: target stream has a length of {0}, expected {1}", remoteLength, _segmentMetadata.Length));
TracingHelper.LogError(ex);
throw ex;
}
}
/// <summary>
/// Validates that the metadata is valid for a resume operation, and also updates the internal Segment States to match what the Server looks like.
/// If any changes are made, the metadata will be saved to its canonical location.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataForResume(UploadMetadata metadata)
{
ValidateMetadataMatchesLocalFile(metadata);
//verify that the target stream does not already exist (in case we don't want to overwrite)
if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath))
{
throw new InvalidOperationException("Target Stream already exists");
}
//make sure we don't upload part of the file as binary, while the rest is non-binary (that's just asking for trouble)
if (this.Parameters.IsBinary != metadata.IsBinary)
{
throw new InvalidOperationException(
string.Format(
"Existing metadata was created for a {0}binary file while the current parameters requested a {1}binary upload.",
metadata.IsBinary ? string.Empty : "non-",
this.Parameters.IsBinary ? string.Empty : "non-"));
}
//see what files(segments) already exist - update metadata accordingly (only for segments that are missing from server; if it's on the server but not in metadata, reupload)
foreach (var segment in metadata.Segments)
{
if (segment.Status == SegmentUploadStatus.Complete)
{
var retryCount = 0;
while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
//verify that the stream exists and that the length is as expected
if (!_frontEnd.StreamExists(segment.Path))
{
// this segment was marked as completed, but no target stream exists; it needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
else
{
var remoteLength = _frontEnd.GetStreamLength(segment.Path);
if (remoteLength != segment.Length)
{
//the target stream has a different length than the input segment, which implies they are inconsistent; it needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
}
break;
}
catch (Exception e)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
throw new UploadFailedException(
string.Format(
"Cannot validate metadata in order to resume due to the following exception retrieving file information: {0}",
e));
}
SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token);
}
}
}
else
{
//anything which is not in 'Completed' status needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
}
metadata.Save();
}
