/// <summary> /// Attempts to load an UploadMetadata object from the given file. /// </summary> /// <param name="filePath">The full path to the file where to load the metadata from</param> /// <returns></returns> /// <exception cref="System.IO.FileNotFoundException">Could not find metadata file</exception> /// <exception cref="Microsoft.Azure.Management.DataLake.StoreUploader.InvalidMetadataException">Unable to parse metadata file</exception> internal static UploadMetadata LoadFrom(string filePath) { if (!File.Exists(filePath)) { throw new FileNotFoundException("Could not find metadata file", filePath); } try { using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { UploadMetadata result = MetadataSerializer.ReadObject(stream) as UploadMetadata; if (result != null) { result.MetadataFilePath = filePath; } return(result); } } catch (Exception ex) { throw new InvalidMetadataException("Unable to parse metadata file", ex); } }
/// <summary> /// Verifies that the metadata is consistent with the local file information. /// </summary> /// <param name="metadata"></param> private void ValidateMetadataMatchesLocalFile(UploadMetadata metadata) { if (metadata.TargetStreamPath.Trim() != this.Parameters.TargetStreamPath.Trim()) { throw new InvalidOperationException("Metadata points to a different target stream than the input parameters"); } //verify that it matches against local file (size, name) var metadataInputFileInfo = new FileInfo(metadata.InputFilePath); var paramInputFileInfo = new FileInfo(this.Parameters.InputFilePath); if (!paramInputFileInfo.FullName.Equals(metadataInputFileInfo.FullName, StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException("The metadata refers to different file than the one requested"); } if (!metadataInputFileInfo.Exists) { throw new InvalidOperationException("The metadata refers to a file that does not exist"); } if (metadata.FileLength != metadataInputFileInfo.Length) { throw new InvalidOperationException("The metadata's file information differs from the actual file"); } }
/// <summary> /// Attempts to load the metadata from an existing file in its canonical location. /// </summary> /// <param name="metadataFilePath">The metadata file path.</param> /// <returns></returns> public UploadMetadata GetExistingMetadata(string metadataFilePath) { //load from file (based on input parameters) var metadata = UploadMetadata.LoadFrom(metadataFilePath); metadata.ValidateConsistency(); return(metadata); }
/// <summary> /// Updates the segment metadata status. /// </summary> /// <param name="metadata">The metadata.</param> /// <param name="segmentNumber">The segment number.</param> /// <param name="newStatus">The new status.</param> private static void UpdateSegmentMetadataStatus(UploadMetadata metadata, int segmentNumber, SegmentUploadStatus newStatus) { metadata.Segments[segmentNumber].Status = newStatus; try { metadata.Save(); } catch { } //no need to crash the program if were unable to save the metadata; it is what's in memory that's important }
/// <summary> /// Uploads the file using the given metadata. /// /// </summary> /// <param name="metadata"></param> private void UploadFile(UploadMetadata metadata) { try { //we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) _previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; ServicePointManager.DefaultConnectionLimit = Math.Max(this.Parameters.ThreadCount, ServicePointManager.DefaultConnectionLimit); //match up the metadata with the information on the server if (this.Parameters.IsResume) { ValidateMetadataForResume(metadata); } else { ValidateMetadataForFreshUpload(metadata); } var segmentProgressTracker = CreateSegmentProgressTracker(metadata); if (metadata.SegmentCount == 0) { // simply create the target stream, overwriting existing streams if they exist _frontEnd.CreateStream(metadata.TargetStreamPath, true, null, 0); } else if (metadata.SegmentCount > 1) { //perform the multi-segment upload var msu = new MultipleSegmentUploader(metadata, this.Parameters.ThreadCount, _frontEnd, _token, segmentProgressTracker); msu.UseSegmentBlockBackOffRetryStrategy = this.Parameters.UseSegmentBlockBackOffRetryStrategy; msu.Upload(); //concatenate the files at the end ConcatenateSegments(metadata); } else { //optimization if we only have one segment: upload it directly to the target stream metadata.Segments[0].Path = metadata.TargetStreamPath; var ssu = new SingleSegmentUploader(0, metadata, _frontEnd, _token, segmentProgressTracker); ssu.UseBackOffRetryStrategy = this.Parameters.UseSegmentBlockBackOffRetryStrategy; ssu.Upload(); } } catch (OperationCanceledException) { // do nothing since we have already marked everything as failed } finally { //revert back the default .NET value for max connections to a host to whatever it was before ServicePointManager.DefaultConnectionLimit = _previousDefaultConnectionLimit; } }
/// <summary> /// Creates a new UploadSegmentMetadata with the given segment number. /// </summary> /// <param name="segmentNumber"></param> /// <param name="metadata"></param> internal UploadSegmentMetadata(int segmentNumber, UploadMetadata metadata) { this.SegmentNumber = segmentNumber; this.Status = SegmentUploadStatus.Pending; string ignored; var targetStreamName = metadata.SplitTargetStreamPathByName(out ignored); this.Path = string.Format("{0}/{1}.{2}.segment{3}", metadata.SegmentStreamDirectory, targetStreamName, metadata.UploadId, this.SegmentNumber); this.Offset = this.SegmentNumber * metadata.SegmentLength; // segment number is zero-based this.Length = CalculateSegmentLength(this.SegmentNumber, metadata); }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="token">The cancellation token to use</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _segmentMetadata = uploadMetadata.Segments[segmentNumber]; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseBackOffRetryStrategy = true; }
/// <summary> /// Verifies that the metadata is valid for a fresh upload. /// </summary> /// <param name="metadata"></param> private void ValidateMetadataForFreshUpload(UploadMetadata metadata) { ValidateMetadataMatchesLocalFile(metadata); //verify that the target stream does not already exist (in case we don't want to overwrite) if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath)) { throw new InvalidOperationException("Target Stream already exists"); } }
/// <summary> /// Creates a new metadata based on the given input parameters, and saves it to its canonical location. /// </summary> /// <param name="metadataFilePath">The metadata file path.</param> /// <returns></returns> public UploadMetadata CreateNewMetadata(string metadataFilePath) { //create metadata var metadata = new UploadMetadata(metadataFilePath, _parameters, _frontend); //save the initial version metadata.Save(); return(metadata); }
/// <summary> /// Gets the pending segments to upload. /// </summary> /// <param name="metadata">The metadata.</param> /// <returns></returns> private static Queue <SegmentQueueItem> GetPendingSegmentsToUpload(UploadMetadata metadata) { var result = new Queue <SegmentQueueItem>(); foreach (var segment in metadata.Segments.Where(segment => segment.Status == SegmentUploadStatus.Pending)) { result.Enqueue(new SegmentQueueItem(segment.SegmentNumber, 0)); } return(result); }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="token">The cancellation token to use.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _maxThreadCount = maxThreadCount; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseSegmentBlockBackOffRetryStrategy = true; }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="token">The cancellation token to use</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _segmentMetadata = uploadMetadata.Segments[segmentNumber]; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseBackOffRetryStrategy = true; }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="token">The cancellation token to use.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _maxThreadCount = maxThreadCount; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseSegmentBlockBackOffRetryStrategy = true; }
/// <summary> /// Aligns segments to match record boundaries (where a record boundary = a new line). /// If not possible (max record size = 4MB), throws an exception. /// </summary> /// <param name="metadata"></param> private void AlignSegmentsToRecordBoundaries(UploadMetadata metadata) { int remainingSegments = 0; using (var stream = new FileStream(metadata.InputFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { long offset = 0; for (int i = 0; i < metadata.Segments.Length; i++) { var segment = metadata.Segments[i]; //updating segment lengths means that both the offset and the length of the next segment needs to be recalculated, to keep the segment lengths somewhat balanced long diff = segment.Offset - offset; segment.Offset = offset; segment.Length += diff; if (segment.Offset >= metadata.FileLength) { continue; } if (segment.SegmentNumber == metadata.Segments.Length - 1) { //last segment picks up the slack segment.Length = metadata.FileLength - segment.Offset; } else { //figure out how much do we need to adjust the length of the segment so it ends on a record boundary (this can be negative or positive) int lengthAdjustment = DetermineLengthAdjustment(segment, stream) + 1; //adjust segment length and offset segment.Length += lengthAdjustment; } offset += segment.Length; remainingSegments++; } } //since we adjusted the segment lengths, it's possible that the last segment(s) became of zero length; so remove it var segments = metadata.Segments; if (remainingSegments < segments.Length) { Array.Resize(ref segments, remainingSegments); metadata.Segments = segments; metadata.SegmentCount = segments.Length; } //NOTE: we are not validating consistency here; this method is called by CreateNewMetadata which calls Save() after this, which validates consistency anyway. }
/// <summary> /// Creates a new metadata based on the given input parameters, and saves it to its canonical location. /// </summary> /// <returns></returns> public UploadMetadata CreateNewMetadata(string metadataFilePath) { //determine segment count, segment length and Upload Id //create metadata var metadata = new UploadMetadata(metadataFilePath, _parameters); if (!_parameters.IsBinary && metadata.SegmentCount > 1) { this.AlignSegmentsToRecordBoundaries(metadata); } //save the initial version metadata.Save(); return(metadata); }
/// <summary> /// Creates a new metadata based on the given input parameters, and saves it to its canonical location. /// </summary> /// <returns></returns> public UploadMetadata CreateNewMetadata(string metadataFilePath) { //determine segment count, segment length and Upload Id //create metadata var metadata = new UploadMetadata(metadataFilePath, _parameters); if (!_parameters.IsBinary && metadata.SegmentCount > 1) { this.AlignSegmentsToRecordBoundaries(metadata); } //save the initial version metadata.Save(); return metadata; }
/// <summary> /// Updates the progress to indicate that a file failed /// </summary> internal void OnFileUploadThreadAborted(UploadMetadata failedFile) { ++this.UploadedFileCount; var previousProgress = _fileProgress.Where(p => p.UploadId.Equals(failedFile.UploadId, StringComparison.InvariantCultureIgnoreCase)).First(); foreach (var segment in previousProgress._segmentProgress) { // only fail out segments that haven't been completed. if (segment.Length != segment.UploadedByteCount) { segment.IsFailed = true; } previousProgress.SetSegmentProgress(segment); } }
/// <summary> /// Creates the segment progress tracker. /// </summary> /// <param name="metadata">The metadata.</param> /// <returns></returns> private IProgress <SegmentUploadProgress> CreateSegmentProgressTracker(UploadMetadata metadata) { if (_progressTracker == null) { return(null); } var overallProgress = new UploadProgress(metadata); return(new Progress <SegmentUploadProgress>( (sup) => { //update the overall progress and report it back overallProgress.SetSegmentProgress(sup); _progressTracker.Report(overallProgress); })); }
/// <summary> /// Calculates the length of the segment with given number for a file with given length that is split into the given number of segments. /// </summary> /// <param name="segmentNumber">The segment number.</param> /// <param name="metadata">The metadata for the current upload.</param> /// <returns></returns> internal static long CalculateSegmentLength(int segmentNumber, UploadMetadata metadata) { if (segmentNumber < 0 || segmentNumber >= metadata.SegmentCount) { throw new ArgumentOutOfRangeException("segmentNumber", "Segment Number must be at least zero and less than the total number of segments"); } if (metadata.FileLength < 0) { throw new ArgumentException("fileLength", "Cannot have a negative file length"); } //verify if the last segment would have a positive value long lastSegmentLength = metadata.FileLength - (metadata.SegmentCount - 1) * metadata.SegmentLength; if (lastSegmentLength < 0) { throw new ArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)"); } else if (lastSegmentLength > metadata.SegmentLength) { //verify if the given segmentCount and segmentLength combination would produce an even split if (metadata.FileLength - (metadata.SegmentCount - 1) * (metadata.SegmentLength + 1) > 0) { throw new ArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength"); } } if (metadata.FileLength == 0) { return(0); } //all segments except the last one have the same length; //the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment Length if (segmentNumber < metadata.SegmentCount - 1) { return(metadata.SegmentLength); } else { return(lastSegmentLength); } }
/// <summary> /// Populates the specified metadata. /// </summary> /// <param name="metadata">The metadata.</param> private void Populate(UploadMetadata metadata) { this.TotalFileLength = metadata.FileLength; this.TotalSegmentCount = metadata.SegmentCount; this.UploadId = metadata.UploadId; _segmentProgress = new SegmentUploadProgress[this.TotalSegmentCount]; foreach (var segmentInfo in metadata.Segments) { if (segmentInfo.Status == SegmentUploadStatus.Complete) { this.UploadedByteCount += segmentInfo.Length; _segmentProgress[segmentInfo.SegmentNumber] = new SegmentUploadProgress(segmentInfo.SegmentNumber, segmentInfo.Length, segmentInfo.Length, false); } else { _segmentProgress[segmentInfo.SegmentNumber] = new SegmentUploadProgress(segmentInfo.SegmentNumber, segmentInfo.Length, 0, false); } } }
/// <summary> /// Uploads the segment. /// </summary> /// <param name="segmentNumber">The segment number.</param> /// <param name="metadata">The metadata.</param> private void UploadSegment(int segmentNumber, UploadMetadata metadata) { //mark the segment as 'InProgress' in the metadata UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.InProgress); var segmentUploader = new SingleSegmentUploader(segmentNumber, metadata, _frontEnd, _token, _progressTracker); segmentUploader.UseBackOffRetryStrategy = this.UseSegmentBlockBackOffRetryStrategy; try { segmentUploader.Upload(); //if we reach this point, the upload was successful; mark it as such UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.Complete); } catch { //something horrible happened, mark the segment as failed and throw the original exception (the caller will handle it) UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.Failed); throw; } }
private void VerifySegmentsAreOnRecordBoundaries(UploadMetadata metadata, string filePath) { using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { foreach (var segment in metadata.Segments) { if (segment.SegmentNumber > 0) { //verify that each segment starts with a non-newline and that the 2 previous characters before that offset are newline characters //2 characters behind: newline stream.Seek(segment.Offset - 2, SeekOrigin.Begin); char c1 = (char)stream.ReadByte(); Assert.True(IsNewline(c1), string.Format("Expecting a newline at offset {0}", stream.Position - 1)); //1 character behind: newline char c2 = (char)stream.ReadByte(); Assert.True(IsNewline(c2), string.Format("Expecting a newline at offset {0}", stream.Position - 1)); //by test design, we never have two consecutive newlines that are the same; we'd always have \r\n, but never \r\r or \r\n var c3 = (char)stream.ReadByte(); Assert.NotEqual(c2, c3); } } } }
/// <summary> /// Constructs a new UploadMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="uploadParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontend">The frontend to use when generating per file metadata.</param> public UploadFolderMetadata(string metadataFilePath, UploadParameters uploadParameters, IFrontEndAdapter frontend) { this.MetadataFilePath = metadataFilePath; this.UploadId = Guid.NewGuid().ToString("N"); this.InputFolderPath = uploadParameters.InputFilePath; this.TargetStreamFolderPath = uploadParameters.TargetStreamPath.TrimEnd('/'); this.IsRecursive = uploadParameters.IsRecursive; // get this list of all files in the source directory, depending on if this is recursive or not. ConcurrentQueue <string> allFiles; ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>(); Dictionary <string, long> downloadFiles = new Dictionary <string, long>(); if (uploadParameters.IsDownload) { foreach (var entry in frontend.ListDirectory(uploadParameters.InputFilePath, uploadParameters.IsRecursive)) { downloadFiles.Add(entry.Key, entry.Value); } allFiles = new ConcurrentQueue <string>(downloadFiles.Keys); this.TotalFileBytes = downloadFiles.Values.Sum(); } else { allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) : Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly)); this.TotalFileBytes = GetByteCountFromFileList(allFiles); } this.FileCount = allFiles.Count(); this.Files = new UploadMetadata[this.FileCount]; // explicitly set the thread pool start amount to at most 500 int threadCount = Math.Min(this.FileCount, 500); var threads = new List <Thread>(threadCount); //start a bunch of new threads that will create the metadata and ensure a protected index. int currentIndex = 0; object indexIncrementLock = new object(); for (int i = 0; i < threadCount; i++) { var t = new Thread(() => { string curFile; while (allFiles.TryDequeue(out curFile)) { try { var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/'); var paramsPerFile = new UploadParameters ( curFile, String.Format("{0}{1}{2}", this.TargetStreamFolderPath, uploadParameters.IsDownload ? "\\" : "/", relativeFilePath), uploadParameters.AccountName, uploadParameters.PerFileThreadCount, uploadParameters.ConcurrentFileCount, uploadParameters.IsOverwrite, uploadParameters.IsResume, uploadParameters.IsBinary, uploadParameters.IsRecursive, uploadParameters.IsDownload, uploadParameters.MaxSegementLength, uploadParameters.LocalMetadataLocation ); long size = -1; if (uploadParameters.IsDownload && downloadFiles != null) { size = downloadFiles[curFile]; } var uploadMetadataPath = Path.Combine(uploadParameters.LocalMetadataLocation, string.Format("{0}.upload.xml", Path.GetFileName(curFile))); var eachFileMetadata = new UploadMetadata(uploadMetadataPath, paramsPerFile, frontend, size); lock (indexIncrementLock) { this.Files[currentIndex] = eachFileMetadata; currentIndex++; } } catch (Exception e) { exceptions.Enqueue(e); } } }); t.Start(); threads.Add(t); } foreach (var t in threads) { t.Join(); } if (exceptions.Count > 0) { throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray()); } }
/// <summary> /// Creates the segment progress tracker. /// </summary> /// <param name="metadata">The metadata.</param> /// <returns></returns> private IProgress<SegmentUploadProgress> CreateSegmentProgressTracker(UploadMetadata metadata) { if (_progressTracker == null) { return null; } var overallProgress = new UploadProgress(metadata); return new Progress<SegmentUploadProgress>( (sup) => { //update the overall progress and report it back overallProgress.SetSegmentProgress(sup); _progressTracker.Report(overallProgress); }); }
/// <summary> /// Concatenates all the segments defined in the metadata into a single stream. /// </summary> /// <param name="metadata"></param> private void ConcatenateSegments(UploadMetadata metadata) { string[] inputPaths = new string[metadata.SegmentCount]; //verify if target stream exists if (_frontEnd.StreamExists(metadata.TargetStreamPath)) { if (this.Parameters.IsOverwrite) { _frontEnd.DeleteStream(metadata.TargetStreamPath); } else { throw new InvalidOperationException("Target Stream already exists"); } } //ensure all input streams exist and are of the expected length //ensure all segments in the metadata are marked as 'complete' var exceptions = new List<Exception>(); Parallel.For( 0, metadata.SegmentCount, new ParallelOptions() { MaxDegreeOfParallelism = this.Parameters.ThreadCount }, (i) => { try { if (metadata.Segments[i].Status != SegmentUploadStatus.Complete) { throw new UploadFailedException("Cannot perform 'Concatenate' operation because not all streams are fully uploaded."); } var remoteStreamPath = metadata.Segments[i].Path; var retryCount = 0; long remoteLength = -1; while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { remoteLength = _frontEnd.GetStreamLength(remoteStreamPath); break; } catch (Exception e) { _token.ThrowIfCancellationRequested(); if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount) { throw new UploadFailedException( string.Format( "Cannot perform 'Concatenate' operation due to the following exception retrieving file information: {0}", e)); } SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token); } } if (remoteLength != metadata.Segments[i].Length) { throw new UploadFailedException(string.Format("Cannot perform 'Concatenate' operation because segment {0} has an incorrect length (expected {1}, actual {2}).", i, metadata.Segments[i].Length, remoteLength)); } inputPaths[i] = remoteStreamPath; } catch (Exception ex) { //collect any exceptions, whether we just generated them above or whether they come from the Front End, exceptions.Add(ex); } }); if (exceptions.Count > 0) { throw new AggregateException("At least one concatenate test failed", exceptions.ToArray()); } //issue the command _frontEnd.Concatenate(metadata.TargetStreamPath, inputPaths); }
public void UploadSegmentMetadata_CalculateParticularSegmentLength() { //verify bad inputs Assert.Throws<ArgumentOutOfRangeException>( () => { UploadSegmentMetadata.CalculateSegmentLength(-1, new UploadMetadata() { FileLength = 10, SegmentCount = 5, SegmentLength = 2 }); }); Assert.Throws<ArgumentOutOfRangeException>( () => { UploadSegmentMetadata.CalculateSegmentLength(100, new UploadMetadata() { FileLength = 10, SegmentCount = 5, SegmentLength = 2 }); }); Assert.Throws<ArgumentException>( () => { UploadSegmentMetadata.CalculateSegmentLength(1, new UploadMetadata() { FileLength = -10, SegmentCount = 5, SegmentLength = 2 }); }); Assert.Throws<ArgumentException>( () => { UploadSegmentMetadata.CalculateSegmentLength(1, new UploadMetadata() { FileLength = 100, SegmentCount = 2, SegmentLength = 2 }); }); Assert.Throws<ArgumentException>( () => { UploadSegmentMetadata.CalculateSegmentLength(1, new UploadMetadata() { FileLength = 100, SegmentCount = 5, SegmentLength = 26 }); }); //test various scenarios with a fixed file length, and varying the segment count from 1 to the FileLength int FileLength = 16 * (int)Math.Pow(2, 20);//16MB for (int segmentCount = 1; segmentCount <= FileLength; segmentCount += 1024) { long typicalSegmentLength = UploadSegmentMetadata.CalculateSegmentLength(FileLength, segmentCount); var uploadMetadata = new UploadMetadata(){FileLength=FileLength,SegmentCount=segmentCount,SegmentLength=typicalSegmentLength}; long firstSegmentLength = UploadSegmentMetadata.CalculateSegmentLength(0, uploadMetadata); long lastSegmentLength = UploadSegmentMetadata.CalculateSegmentLength(segmentCount - 1, uploadMetadata); Assert.Equal(typicalSegmentLength, firstSegmentLength); if (segmentCount == 1) { Assert.Equal(firstSegmentLength, lastSegmentLength); } long reconstructedFileLength = typicalSegmentLength * (segmentCount - 1) + lastSegmentLength; Assert.Equal(FileLength, reconstructedFileLength); } }
/// <summary> /// Calculates the length of the segment with given number for a file with given length that is split into the given number of segments. /// </summary> /// <param name="segmentNumber">The segment number.</param> /// <param name="metadata">The metadata for the current upload.</param> /// <returns></returns> internal static long CalculateSegmentLength(int segmentNumber, UploadMetadata metadata) { if (segmentNumber < 0 || segmentNumber >= metadata.SegmentCount) { throw new ArgumentOutOfRangeException("segmentNumber", "Segment Number must be at least zero and less than the total number of segments"); } if (metadata.FileLength < 0) { throw new ArgumentException("fileLength", "Cannot have a negative file length"); } //verify if the last segment would have a positive value long lastSegmentLength = metadata.FileLength - (metadata.SegmentCount - 1) * metadata.SegmentLength; if (lastSegmentLength < 0) { throw new ArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)"); } else if (lastSegmentLength > metadata.SegmentLength) { //verify if the given segmentCount and segmentLength combination would produce an even split if (metadata.FileLength - (metadata.SegmentCount - 1) * (metadata.SegmentLength + 1) > 0) { throw new ArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength"); } } if (metadata.FileLength == 0) { return 0; } //all segments except the last one have the same length; //the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment Length if (segmentNumber < metadata.SegmentCount - 1) { return metadata.SegmentLength; } else { return lastSegmentLength; } }
/// <summary> /// Gets the pending segments to upload. /// </summary> /// <param name="metadata">The metadata.</param> /// <returns></returns> private static Queue<SegmentQueueItem> GetPendingSegmentsToUpload(UploadMetadata metadata) { var result = new Queue<SegmentQueueItem>(); foreach (var segment in metadata.Segments.Where(segment => segment.Status == SegmentUploadStatus.Pending)) { result.Enqueue(new SegmentQueueItem(segment.SegmentNumber, 0)); } return result; }
/// <summary> /// Concatenates all the segments defined in the metadata into a single stream. /// </summary> /// <param name="metadata"></param> private void ConcatenateSegments(UploadMetadata metadata) { string[] inputPaths = new string[metadata.SegmentCount]; //verify if target stream exists if (_frontEnd.StreamExists(metadata.TargetStreamPath)) { if (this.Parameters.IsOverwrite) { _frontEnd.DeleteStream(metadata.TargetStreamPath); } else { throw new InvalidOperationException("Target Stream already exists"); } } //ensure all input streams exist and are of the expected length //ensure all segments in the metadata are marked as 'complete' var exceptions = new List <Exception>(); Parallel.For( 0, metadata.SegmentCount, new ParallelOptions() { MaxDegreeOfParallelism = this.Parameters.ThreadCount }, (i) => { try { if (metadata.Segments[i].Status != SegmentUploadStatus.Complete) { throw new UploadFailedException("Cannot perform 'Concatenate' operation because not all streams are fully uploaded."); } var remoteStreamPath = metadata.Segments[i].Path; var retryCount = 0; long remoteLength = -1; while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { remoteLength = _frontEnd.GetStreamLength(remoteStreamPath); break; } catch (Exception e) { _token.ThrowIfCancellationRequested(); if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount) { throw new UploadFailedException( string.Format( "Cannot perform 'Concatenate' operation due to the following exception retrieving file information: {0}", e)); } SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token); } } if (remoteLength != metadata.Segments[i].Length) { throw new UploadFailedException(string.Format("Cannot perform 'Concatenate' operation because segment {0} has an incorrect length (expected {1}, actual {2}).", i, metadata.Segments[i].Length, remoteLength)); } inputPaths[i] = remoteStreamPath; } catch (Exception ex) { //collect any exceptions, whether we just generated them above or whether they come from the Front End, exceptions.Add(ex); } }); if (exceptions.Count > 0) { throw new AggregateException("At least one concatenate test failed", exceptions.ToArray()); } //issue the command _frontEnd.Concatenate(metadata.TargetStreamPath, inputPaths); }
private void VerifyTargetStreamIsComplete(UploadSegmentMetadata segmentMetadata, UploadMetadata metadata, InMemoryFrontEnd frontEnd) { Assert.Equal(SegmentUploadStatus.Complete, segmentMetadata.Status); Assert.True(frontEnd.StreamExists(segmentMetadata.Path), string.Format("Segment {0} was not uploaded", segmentMetadata.SegmentNumber)); Assert.Equal(segmentMetadata.Length, frontEnd.GetStreamLength(segmentMetadata.Path)); var actualContents = frontEnd.GetStreamContents(segmentMetadata.Path); var expectedContents = GetExpectedContents(segmentMetadata, metadata); AssertExtensions.AreEqual(expectedContents, actualContents, "Segment {0} has unexpected contents", segmentMetadata.SegmentNumber); }
/// <summary> /// Initializes a new instance of the <see cref="UploadProgress" /> class. /// </summary> /// <param name="metadata">The metadata.</param> /// <param name="progressTracker">The progress tracker.</param> internal UploadProgress(UploadMetadata metadata, IProgress <UploadProgress> progressTracker = null) { _progressTracker = progressTracker; Populate(metadata); }
private UploadMetadata CreateMetadata(int segmentCount) { var path = Path.GetTempFileName(); var metadata = new UploadMetadata() { MetadataFilePath = path, InputFilePath = _smallFilePath, FileLength = _smallFileContents.Length, SegmentCount = segmentCount, SegmentLength = UploadSegmentMetadata.CalculateSegmentLength(_smallFileContents.Length, segmentCount), Segments = new UploadSegmentMetadata[segmentCount], TargetStreamPath = "abc", UploadId = "123", IsBinary = true }; long offset = 0; for (int i = 0; i < segmentCount; i++) { long length = UploadSegmentMetadata.CalculateSegmentLength(i, metadata); metadata.Segments[i] = new UploadSegmentMetadata() { SegmentNumber = i, Offset = offset, Status = SegmentUploadStatus.Pending, Length = length, Path = string.Format("{0}.{1}.segment{2}", metadata.TargetStreamPath, metadata.UploadId, i) }; offset += length; } return metadata; }
private byte[] GetExpectedContents(UploadSegmentMetadata segment, UploadMetadata metadata) { byte[] result = new byte[segment.Length]; Array.Copy(_smallFileContents, segment.SegmentNumber * metadata.SegmentLength, result, 0, segment.Length); return result; }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) : this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker) { }
private void VerifyTargetStreamsAreComplete(UploadMetadata metadata, InMemoryFrontEnd fe) { foreach (var segment in metadata.Segments) { VerifyTargetStreamIsComplete(segment, metadata, fe); } }
/// <summary> /// Validates that the metadata is valid for a resume operation, and also updates the internal Segment States to match what the Server looks like. /// If any changes are made, the metadata will be saved to its canonical location. /// </summary> /// <param name="metadata"></param> private void ValidateMetadataForResume(UploadMetadata metadata) { ValidateMetadataMatchesLocalFile(metadata); //verify that the target stream does not already exist (in case we don't want to overwrite) if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath)) { throw new InvalidOperationException("Target Stream already exists"); } //make sure we don't upload part of the file as binary, while the rest is non-binary (that's just asking for trouble) if (this.Parameters.IsBinary != metadata.IsBinary) { throw new InvalidOperationException( string.Format( "Existing metadata was created for a {0}binary file while the current parameters requested a {1}binary upload.", metadata.IsBinary ? string.Empty : "non-", this.Parameters.IsBinary ? string.Empty : "non-")); } //see what files(segments) already exist - update metadata accordingly (only for segments that are missing from server; if it's on the server but not in metadata, reupload) foreach (var segment in metadata.Segments) { if (segment.Status == SegmentUploadStatus.Complete) { var retryCount = 0; while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount) { _token.ThrowIfCancellationRequested(); retryCount++; try { //verify that the stream exists and that the length is as expected if (!_frontEnd.StreamExists(segment.Path)) { // this segment was marked as completed, but no target stream exists; it needs to be reuploaded segment.Status = SegmentUploadStatus.Pending; } else { var remoteLength = _frontEnd.GetStreamLength(segment.Path); if (remoteLength != segment.Length) { //the target stream has a different length than the input segment, which implies they are inconsistent; it needs to be reuploaded segment.Status = SegmentUploadStatus.Pending; } } break; } catch (Exception e) { _token.ThrowIfCancellationRequested(); if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount) { throw new UploadFailedException( string.Format( "Cannot validate metadata in order to resume due to the following exception retrieving file information: {0}", e)); } SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token); } } } else { //anything which is not in 'Completed' status needs to be reuploaded segment.Status = SegmentUploadStatus.Pending; } } metadata.Save(); }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress <SegmentUploadProgress> progressTracker = null) : this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentUploadProgress> progressTracker = null) : this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) : this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker) { }