/// <summary> /// Uploads the segment. /// </summary> /// <param name="segmentNumber">The segment number.</param> /// <param name="metadata">The metadata.</param> private void UploadSegment(int segmentNumber, TransferMetadata metadata) { //mark the segment as 'InProgress' in the metadata UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentTransferStatus.InProgress); var segmentUploader = new SingleSegmentUploader(segmentNumber, metadata, _frontEnd, _token, _progressTracker); segmentUploader.UseBackOffRetryStrategy = this.UseSegmentBlockBackOffRetryStrategy; try { segmentUploader.Upload(); //if we reach this point, the upload was successful; mark it as such UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentTransferStatus.Complete); } catch (Exception ex) { //something horrible happened, mark the segment as failed and throw the original exception (the caller will handle it) UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentTransferStatus.Failed); TracingHelper.LogError(ex); throw ex; } }
/// <summary> /// Attempts to load the metadata from an existing file in its canonical location. /// </summary> /// <param name="metadataFilePath">The metadata file path.</param> /// <returns></returns> public TransferMetadata GetExistingMetadata(string metadataFilePath) { //load from file (based on input parameters) var metadata = TransferMetadata.LoadFrom(metadataFilePath); metadata.ValidateConsistency(); return(metadata); }
/// <summary> /// Updates the segment metadata status. /// </summary> /// <param name="metadata">The metadata.</param> /// <param name="segmentNumber">The segment number.</param> /// <param name="newStatus">The new status.</param> private static void UpdateSegmentMetadataStatus(TransferMetadata metadata, int segmentNumber, SegmentTransferStatus newStatus) { metadata.Segments[segmentNumber].Status = newStatus; try { metadata.Save(); } catch { } //no need to crash the program if were unable to save the metadata; it is what's in memory that's important }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="token">The cancellation token to use.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null) { _metadata = uploadMetadata; _maxThreadCount = maxThreadCount; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseSegmentBlockBackOffRetryStrategy = true; }
/// <summary> /// Gets the pending segments to upload. /// </summary> /// <param name="metadata">The metadata.</param> /// <returns></returns> private static Queue <SegmentQueueItem> GetPendingSegmentsToUpload(TransferMetadata metadata) { var result = new Queue <SegmentQueueItem>(); foreach (var segment in metadata.Segments.Where(segment => segment.Status == SegmentTransferStatus.Pending)) { result.Enqueue(new SegmentQueueItem(segment.SegmentNumber, 0)); } return(result); }
/// <summary> /// Creates a new metadata based on the given input parameters, and saves it to its canonical location. /// </summary> /// <param name="metadataFilePath">The metadata file path.</param> /// <returns></returns> public TransferMetadata CreateNewMetadata(string metadataFilePath) { //create metadata var metadata = new TransferMetadata(metadataFilePath, _parameters, _frontend); //save the initial version metadata.Save(); return(metadata); }
/// <summary> /// Creates a new downloader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="downloadMetadata">The metadata for the entire download.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="token">The cancellation token to use</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null) { _metadata = downloadMetadata; _segmentMetadata = downloadMetadata.Segments[segmentNumber]; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseBackOffRetryStrategy = true; }
/// <summary> /// Creates a new UploadSegmentMetadata with the given segment number. /// </summary> /// <param name="segmentNumber"></param> /// <param name="metadata"></param> internal TransferSegmentMetadata(int segmentNumber, TransferMetadata metadata) { this.SegmentNumber = segmentNumber; this.Status = SegmentTransferStatus.Pending; string ignored; var targetStreamName = metadata.SplitTargetStreamPathByName(out ignored); this.Path = string.Format("{0}/{1}.{2}.segment{3}", metadata.SegmentStreamDirectory, targetStreamName, metadata.TransferId, this.SegmentNumber); this.Offset = this.SegmentNumber * metadata.SegmentLength; // segment number is zero-based this.Length = CalculateSegmentLength(this.SegmentNumber, metadata); }
/// <summary> /// Updates the progress to indicate that a file failed /// </summary> internal void OnFileTransferThreadAborted(TransferMetadata failedFile) { ++this.FailedFileCount; var previousProgress = _fileProgress.Where(p => p.TransferId.Equals(failedFile.TransferId, StringComparison.OrdinalIgnoreCase)).First(); foreach (var segment in previousProgress._segmentProgress) { // only fail out segments that haven't been completed. if (segment.Length != segment.TransferredByteCount) { segment.IsFailed = true; } previousProgress.SetSegmentProgress(segment); } }
/// <summary> /// Calculates the length of the segment with given number for a file with given length that is split into the given number of segments. /// </summary> /// <param name="segmentNumber">The segment number.</param> /// <param name="metadata">The metadata for the current upload.</param> /// <returns></returns> internal static long CalculateSegmentLength(int segmentNumber, TransferMetadata metadata) { if (segmentNumber < 0 || segmentNumber >= metadata.SegmentCount) { throw new ArgumentOutOfRangeException("segmentNumber", "Segment Number must be at least zero and less than the total number of segments"); } if (metadata.FileLength < 0) { throw new ArgumentException("fileLength", "Cannot have a negative file length"); } //verify if the last segment would have a positive value long lastSegmentLength = metadata.FileLength - (metadata.SegmentCount - 1) * metadata.SegmentLength; if (lastSegmentLength < 0) { throw new ArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)"); } else if (lastSegmentLength > metadata.SegmentLength) { //verify if the given segmentCount and segmentLength combination would produce an even split if (metadata.FileLength - (metadata.SegmentCount - 1) * (metadata.SegmentLength + 1) > 0) { throw new ArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength"); } } if (metadata.FileLength == 0) { return(0); } //all segments except the last one have the same length; //the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment Length if (segmentNumber < metadata.SegmentCount - 1) { return(metadata.SegmentLength); } else { return(lastSegmentLength); } }
/// <summary> /// Populates the specified metadata. /// </summary> /// <param name="metadata">The metadata.</param> private void Populate(TransferMetadata metadata) { this.TotalFileLength = metadata.FileLength; this.TotalSegmentCount = metadata.SegmentCount; this.TransferId = metadata.TransferId; _segmentProgress = new SegmentTransferProgress[this.TotalSegmentCount]; foreach (var segmentInfo in metadata.Segments) { if (segmentInfo.Status == SegmentTransferStatus.Complete) { this.TransferredByteCount += segmentInfo.Length; _segmentProgress[segmentInfo.SegmentNumber] = new SegmentTransferProgress(segmentInfo.SegmentNumber, segmentInfo.Length, segmentInfo.Length, false); } else { _segmentProgress[segmentInfo.SegmentNumber] = new SegmentTransferProgress(segmentInfo.SegmentNumber, segmentInfo.Length, 0, false); } } }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) : this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Constructs a new TransferFolderMetadata object from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="transferParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontend">The frontend to use when generating per file metadata.</param> public TransferFolderMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontend) { this.MetadataFilePath = metadataFilePath; this.TransferId = Guid.NewGuid().ToString("N"); this.InputFolderPath = transferParameters.InputFilePath; this.TargetStreamFolderPath = transferParameters.TargetStreamPath.TrimEnd('/'); this.IsRecursive = transferParameters.IsRecursive; // get this list of all files in the source directory, depending on if this is recursive or not. ConcurrentQueue <string> allFiles; ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>(); Dictionary <string, long> downloadFiles = new Dictionary <string, long>(); if (transferParameters.IsDownload) { foreach (var entry in frontend.ListDirectory(transferParameters.InputFilePath, transferParameters.IsRecursive)) { downloadFiles.Add(entry.Key, entry.Value); } allFiles = new ConcurrentQueue <string>(downloadFiles.Keys); this.TotalFileBytes = downloadFiles.Values.Sum(); } else { allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) : Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly)); this.TotalFileBytes = GetByteCountFromFileList(allFiles); } this.FileCount = allFiles.Count(); this.Files = new TransferMetadata[this.FileCount]; // explicitly set the thread pool start amount to at most 500 int threadCount = Math.Min(this.FileCount, 500); var threads = new List <Thread>(threadCount); //start a bunch of new threads that will create the metadata and ensure a protected index. int currentIndex = 0; object indexIncrementLock = new object(); for (int i = 0; i < threadCount; i++) { var t = new Thread(() => { string curFile; while (allFiles.TryDequeue(out curFile)) { try { var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/'); var paramsPerFile = new TransferParameters ( curFile, String.Format("{0}{1}{2}", this.TargetStreamFolderPath, transferParameters.IsDownload ? "\\" : "/", relativeFilePath), transferParameters.AccountName, transferParameters.PerFileThreadCount, transferParameters.ConcurrentFileCount, transferParameters.IsOverwrite, transferParameters.IsResume, transferParameters.IsBinary, transferParameters.IsRecursive, transferParameters.IsDownload, transferParameters.MaxSegementLength, transferParameters.LocalMetadataLocation ); long size = -1; if (transferParameters.IsDownload && downloadFiles != null) { size = downloadFiles[curFile]; } var transferMetadataPath = Path.Combine(transferParameters.LocalMetadataLocation, string.Format("{0}.transfer.xml", Path.GetFileName(curFile))); var eachFileMetadata = new TransferMetadata(transferMetadataPath, paramsPerFile, frontend, size); lock (indexIncrementLock) { this.Files[currentIndex] = eachFileMetadata; currentIndex++; } } catch (Exception e) { exceptions.Enqueue(e); } } }); t.Start(); threads.Add(t); } foreach (var t in threads) { t.Join(); } if (exceptions.Count > 0) { throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray()); } }
/// <summary> /// Creates a new downloader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="downloadMetadata">The metadata for the entire download.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) : this(segmentNumber, downloadMetadata, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Initializes a new instance of the <see cref="TransferProgress" /> class. /// </summary> /// <param name="metadata">The metadata.</param> /// <param name="progressTracker">The progress tracker.</param> internal TransferProgress(TransferMetadata metadata, IProgress <TransferProgress> progressTracker = null) { _progressTracker = progressTracker; Populate(metadata); }