/// <summary> /// Verifies the uploaded stream. /// </summary> /// <exception cref="UploadFailedException"></exception> private void VerifyUploadedStream() { //verify that the remote stream has the length we expected. var retryCount = 0; long remoteLength = -1; while (retryCount < MaxBufferUploadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { remoteLength = _frontEnd.GetStreamLength(_segmentMetadata.Path); break; } catch (Exception) { _token.ThrowIfCancellationRequested(); if (retryCount >= MaxBufferUploadAttemptCount) { throw; } WaitForRetry(retryCount, this.UseBackOffRetryStrategy, _token); } } if (_segmentMetadata.Length != remoteLength) { throw new UploadFailedException(string.Format("Post-upload stream verification failed: target stream has a length of {0}, expected {1}", remoteLength, _segmentMetadata.Length)); } }
/// <summary> /// Constructs a new TransferMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="transferParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontEnd">The front end. This is used only in the constructor for determining file length</param> internal TransferMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontEnd, long fileSize = -1) { this.MetadataFilePath = metadataFilePath; this.TransferId = Guid.NewGuid().ToString("N"); this.InputFilePath = transferParameters.InputFilePath; this.TargetStreamPath = transferParameters.TargetStreamPath; this.IsDownload = transferParameters.IsDownload; this.SegmentStreamDirectory = GetSegmentStreamDirectory(); this.IsBinary = transferParameters.IsBinary; this.FileLength = fileSize < 0 ? frontEnd.GetStreamLength(transferParameters.InputFilePath, !IsDownload) : fileSize; this.EncodingCodePage = transferParameters.FileEncoding.CodePage; // we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic. // this protects us against agressive increase of thread count resulting in far more segments than // is reasonable for a given file size. We also ensure that each segment is at least 256mb in size. // This is the size that ensures we have the optimal storage creation in the store. var preliminarySegmentCount = (int)Math.Ceiling((double)this.FileLength / transferParameters.MaxSegementLength); this.SegmentCount = Math.Min(preliminarySegmentCount, TransferSegmentMetadata.CalculateSegmentCount(this.FileLength)); this.SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(this.FileLength, this.SegmentCount); this.Segments = new TransferSegmentMetadata[this.SegmentCount]; for (int i = 0; i < this.SegmentCount; i++) { this.Segments[i] = new TransferSegmentMetadata(i, this); } if (!transferParameters.IsBinary && this.SegmentCount > 1 && !this.IsDownload) { this.AlignSegmentsToRecordBoundaries(); // ensure that nothing strange happened during alignment this.ValidateConsistency(); } // initialize the status to pending, since it is not yet done. this.Status = SegmentTransferStatus.Pending; }
/// <summary> /// Verifies the downloaded stream. /// </summary> /// <exception cref="TransferFailedException"></exception> internal void VerifyDownloadedStream() { //verify that the remote stream has the length we expected. var retryCount = 0; long remoteLength = -1; while (retryCount < MaxBufferDownloadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { remoteLength = _frontEnd.GetStreamLength(_segmentMetadata.Path, _metadata.IsDownload); break; } catch (Exception e) { _token.ThrowIfCancellationRequested(); if (retryCount >= MaxBufferDownloadAttemptCount) { TracingHelper.LogError(e); throw e; } var waitTime = WaitForRetry(retryCount, this.UseBackOffRetryStrategy, _token); TracingHelper.LogInfo("VerifyDownloadedStream: GetStreamLength at path:{0} failed on try: {1} with exception: {2}. Wait time in ms before retry: {3}", _segmentMetadata.Path, retryCount, e, waitTime); } } if (_segmentMetadata.Length != remoteLength) { var ex = new TransferFailedException(string.Format("Post-download stream verification failed: target stream has a length of {0}, expected {1}", remoteLength, _segmentMetadata.Length)); TracingHelper.LogError(ex); throw ex; } }
/// <summary> /// Validates that the metadata is valid for a resume operation, and also updates the internal Segment States to match what the Server looks like. /// If any changes are made, the metadata will be saved to its canonical location. /// </summary> /// <param name="metadata"></param> private void ValidateMetadataForResume(UploadMetadata metadata) { ValidateMetadataMatchesLocalFile(metadata); //verify that the target stream does not already exist (in case we don't want to overwrite) if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath)) { throw new InvalidOperationException("Target Stream already exists"); } //make sure we don't upload part of the file as binary, while the rest is non-binary (that's just asking for trouble) if (this.Parameters.IsBinary != metadata.IsBinary) { throw new InvalidOperationException( string.Format( "Existing metadata was created for a {0}binary file while the current parameters requested a {1}binary upload.", metadata.IsBinary ? string.Empty : "non-", this.Parameters.IsBinary ? string.Empty : "non-")); } //see what files(segments) already exist - update metadata accordingly (only for segments that are missing from server; if it's on the server but not in metadata, reupload) foreach (var segment in metadata.Segments) { if (segment.Status == SegmentUploadStatus.Complete) { var retryCount = 0; while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { //verify that the stream exists and that the length is as expected if (!_frontEnd.StreamExists(segment.Path)) { // this segment was marked as completed, but no target stream exists; it needs to be reuploaded segment.Status = SegmentUploadStatus.Pending; } else { var remoteLength = _frontEnd.GetStreamLength(segment.Path); if (remoteLength != segment.Length) { //the target stream has a different length than the input segment, which implies they are inconsistent; it needs to be reuploaded segment.Status = SegmentUploadStatus.Pending; } } break; } catch (Exception e) { _token.ThrowIfCancellationRequested(); if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount) { throw new UploadFailedException( string.Format( "Cannot validate metadata in order to resume due to the following exception retrieving file information: {0}", e)); } SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token); } } } else { //anything which is not in 'Completed' status needs to be reuploaded segment.Status = SegmentUploadStatus.Pending; } } metadata.Save(); }