/// <summary>
/// Uploads the segment.
/// </summary>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="metadata">The metadata.</param>
private void UploadSegment(int segmentNumber, TransferMetadata metadata)
{
//mark the segment as 'InProgress' in the metadata
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentTransferStatus.InProgress);
var segmentUploader = new SingleSegmentUploader(segmentNumber, metadata, _frontEnd, _token, _progressTracker);
segmentUploader.UseBackOffRetryStrategy = this.UseSegmentBlockBackOffRetryStrategy;
try
{
segmentUploader.Upload();
//if we reach this point, the upload was successful; mark it as such
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentTransferStatus.Complete);
}
catch (Exception ex)
{
//something horrible happened, mark the segment as failed and throw the original exception (the caller will handle it)
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentTransferStatus.Failed);
TracingHelper.LogError(ex);
throw ex;
}
}
/// <summary>
/// Attempts to load the metadata from an existing file in its canonical location.
/// </summary>
/// <param name="metadataFilePath">The metadata file path.</param>
/// <returns></returns>
public TransferMetadata GetExistingMetadata(string metadataFilePath)
{
//load from file (based on input parameters)
var metadata = TransferMetadata.LoadFrom(metadataFilePath);
metadata.ValidateConsistency();
return(metadata);
}
/// <summary>
/// Updates the segment metadata status.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="newStatus">The new status.</param>
private static void UpdateSegmentMetadataStatus(TransferMetadata metadata, int segmentNumber, SegmentTransferStatus newStatus)
{
metadata.Segments[segmentNumber].Status = newStatus;
try
{
metadata.Save();
}
catch { } //no need to crash the program if were unable to save the metadata; it is what's in memory that's important
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="token">The cancellation token to use.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_maxThreadCount = maxThreadCount;
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseSegmentBlockBackOffRetryStrategy = true;
}
/// <summary>
/// Gets the pending segments to upload.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <returns></returns>
private static Queue <SegmentQueueItem> GetPendingSegmentsToUpload(TransferMetadata metadata)
{
var result = new Queue <SegmentQueueItem>();
foreach (var segment in metadata.Segments.Where(segment => segment.Status == SegmentTransferStatus.Pending))
{
result.Enqueue(new SegmentQueueItem(segment.SegmentNumber, 0));
}
return(result);
}
/// <summary>
/// Creates a new metadata based on the given input parameters, and saves it to its canonical location.
/// </summary>
/// <param name="metadataFilePath">The metadata file path.</param>
/// <returns></returns>
public TransferMetadata CreateNewMetadata(string metadataFilePath)
{
//create metadata
var metadata = new TransferMetadata(metadataFilePath, _parameters, _frontend);
//save the initial version
metadata.Save();
return(metadata);
}
/// <summary>
/// Creates a new downloader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="downloadMetadata">The metadata for the entire download.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="token">The cancellation token to use</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null)
{
_metadata = downloadMetadata;
_segmentMetadata = downloadMetadata.Segments[segmentNumber];
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new UploadSegmentMetadata with the given segment number.
/// </summary>
/// <param name="segmentNumber"></param>
/// <param name="metadata"></param>
internal TransferSegmentMetadata(int segmentNumber, TransferMetadata metadata)
{
this.SegmentNumber = segmentNumber;
this.Status = SegmentTransferStatus.Pending;
string ignored;
var targetStreamName = metadata.SplitTargetStreamPathByName(out ignored);
this.Path = string.Format("{0}/{1}.{2}.segment{3}", metadata.SegmentStreamDirectory, targetStreamName, metadata.TransferId, this.SegmentNumber);
this.Offset = this.SegmentNumber * metadata.SegmentLength; // segment number is zero-based
this.Length = CalculateSegmentLength(this.SegmentNumber, metadata);
}
/// <summary>
/// Updates the progress to indicate that a file failed
/// </summary>
internal void OnFileTransferThreadAborted(TransferMetadata failedFile)
{
++this.FailedFileCount;
var previousProgress = _fileProgress.Where(p => p.TransferId.Equals(failedFile.TransferId, StringComparison.OrdinalIgnoreCase)).First();
foreach (var segment in previousProgress._segmentProgress)
{
// only fail out segments that haven't been completed.
if (segment.Length != segment.TransferredByteCount)
{
segment.IsFailed = true;
}
previousProgress.SetSegmentProgress(segment);
}
}
/// <summary>
/// Calculates the length of the segment with given number for a file with given length that is split into the given number of segments.
/// </summary>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="metadata">The metadata for the current upload.</param>
/// <returns></returns>
internal static long CalculateSegmentLength(int segmentNumber, TransferMetadata metadata)
{
if (segmentNumber < 0 || segmentNumber >= metadata.SegmentCount)
{
throw new ArgumentOutOfRangeException("segmentNumber", "Segment Number must be at least zero and less than the total number of segments");
}
if (metadata.FileLength < 0)
{
throw new ArgumentException("fileLength", "Cannot have a negative file length");
}
//verify if the last segment would have a positive value
long lastSegmentLength = metadata.FileLength - (metadata.SegmentCount - 1) * metadata.SegmentLength;
if (lastSegmentLength < 0)
{
throw new ArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)");
}
else if (lastSegmentLength > metadata.SegmentLength)
{
//verify if the given segmentCount and segmentLength combination would produce an even split
if (metadata.FileLength - (metadata.SegmentCount - 1) * (metadata.SegmentLength + 1) > 0)
{
throw new ArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength");
}
}
if (metadata.FileLength == 0)
{
return(0);
}
//all segments except the last one have the same length;
//the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment Length
if (segmentNumber < metadata.SegmentCount - 1)
{
return(metadata.SegmentLength);
}
else
{
return(lastSegmentLength);
}
}
/// <summary>
/// Populates the specified metadata.
/// </summary>
/// <param name="metadata">The metadata.</param>
private void Populate(TransferMetadata metadata)
{
this.TotalFileLength = metadata.FileLength;
this.TotalSegmentCount = metadata.SegmentCount;
this.TransferId = metadata.TransferId;
_segmentProgress = new SegmentTransferProgress[this.TotalSegmentCount];
foreach (var segmentInfo in metadata.Segments)
{
if (segmentInfo.Status == SegmentTransferStatus.Complete)
{
this.TransferredByteCount += segmentInfo.Length;
_segmentProgress[segmentInfo.SegmentNumber] = new SegmentTransferProgress(segmentInfo.SegmentNumber, segmentInfo.Length, segmentInfo.Length, false);
}
else
{
_segmentProgress[segmentInfo.SegmentNumber] = new SegmentTransferProgress(segmentInfo.SegmentNumber, segmentInfo.Length, 0, false);
}
}
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) :
this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Constructs a new TransferFolderMetadata object from the given parameters.
/// </summary>
/// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param>
/// <param name="transferParameters">The parameters to use for constructing this metadata.</param>
/// <param name="frontend">The frontend to use when generating per file metadata.</param>
public TransferFolderMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontend)
{
this.MetadataFilePath = metadataFilePath;
this.TransferId = Guid.NewGuid().ToString("N");
this.InputFolderPath = transferParameters.InputFilePath;
this.TargetStreamFolderPath = transferParameters.TargetStreamPath.TrimEnd('/');
this.IsRecursive = transferParameters.IsRecursive;
// get this list of all files in the source directory, depending on if this is recursive or not.
ConcurrentQueue <string> allFiles;
ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>();
Dictionary <string, long> downloadFiles = new Dictionary <string, long>();
if (transferParameters.IsDownload)
{
foreach (var entry in frontend.ListDirectory(transferParameters.InputFilePath, transferParameters.IsRecursive))
{
downloadFiles.Add(entry.Key, entry.Value);
}
allFiles = new ConcurrentQueue <string>(downloadFiles.Keys);
this.TotalFileBytes = downloadFiles.Values.Sum();
}
else
{
allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) :
Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly));
this.TotalFileBytes = GetByteCountFromFileList(allFiles);
}
this.FileCount = allFiles.Count();
this.Files = new TransferMetadata[this.FileCount];
// explicitly set the thread pool start amount to at most 500
int threadCount = Math.Min(this.FileCount, 500);
var threads = new List <Thread>(threadCount);
//start a bunch of new threads that will create the metadata and ensure a protected index.
int currentIndex = 0;
object indexIncrementLock = new object();
for (int i = 0; i < threadCount; i++)
{
var t = new Thread(() => {
string curFile;
while (allFiles.TryDequeue(out curFile))
{
try
{
var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/');
var paramsPerFile = new TransferParameters
(
curFile,
String.Format("{0}{1}{2}", this.TargetStreamFolderPath, transferParameters.IsDownload ? "\\" : "/", relativeFilePath),
transferParameters.AccountName,
transferParameters.PerFileThreadCount,
transferParameters.ConcurrentFileCount,
transferParameters.IsOverwrite,
transferParameters.IsResume,
transferParameters.IsBinary,
transferParameters.IsRecursive,
transferParameters.IsDownload,
transferParameters.MaxSegementLength,
transferParameters.LocalMetadataLocation
);
long size = -1;
if (transferParameters.IsDownload && downloadFiles != null)
{
size = downloadFiles[curFile];
}
var transferMetadataPath = Path.Combine(transferParameters.LocalMetadataLocation, string.Format("{0}.transfer.xml", Path.GetFileName(curFile)));
var eachFileMetadata = new TransferMetadata(transferMetadataPath, paramsPerFile, frontend, size);
lock (indexIncrementLock)
{
this.Files[currentIndex] = eachFileMetadata;
currentIndex++;
}
}
catch (Exception e)
{
exceptions.Enqueue(e);
}
}
});
t.Start();
threads.Add(t);
}
foreach (var t in threads)
{
t.Join();
}
if (exceptions.Count > 0)
{
throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray());
}
}
/// <summary>
/// Creates a new downloader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="downloadMetadata">The metadata for the entire download.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) :
this(segmentNumber, downloadMetadata, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="TransferProgress" /> class.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <param name="progressTracker">The progress tracker.</param>
internal TransferProgress(TransferMetadata metadata, IProgress <TransferProgress> progressTracker = null)
{
_progressTracker = progressTracker;
Populate(metadata);
}
