/// <summary>
/// Attempts to load an UploadMetadata object from the given file.
/// </summary>
/// <param name="filePath">The full path to the file where to load the metadata from</param>
/// <returns></returns>
/// <exception cref="System.IO.FileNotFoundException">Could not find metadata file</exception>
/// <exception cref="Microsoft.Azure.Management.DataLake.StoreUploader.InvalidMetadataException">Unable to parse metadata file</exception>
internal static UploadMetadata LoadFrom(string filePath)
{
if (!File.Exists(filePath))
{
throw new FileNotFoundException("Could not find metadata file", filePath);
}
try
{
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
UploadMetadata result = MetadataSerializer.ReadObject(stream) as UploadMetadata;
if (result != null)
{
result.MetadataFilePath = filePath;
}
return(result);
}
}
catch (Exception ex)
{
throw new InvalidMetadataException("Unable to parse metadata file", ex);
}
}
/// <summary>
/// Verifies that the metadata is consistent with the local file information.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataMatchesLocalFile(UploadMetadata metadata)
{
if (metadata.TargetStreamPath.Trim() != this.Parameters.TargetStreamPath.Trim())
{
throw new InvalidOperationException("Metadata points to a different target stream than the input parameters");
}
//verify that it matches against local file (size, name)
var metadataInputFileInfo = new FileInfo(metadata.InputFilePath);
var paramInputFileInfo = new FileInfo(this.Parameters.InputFilePath);
if (!paramInputFileInfo.FullName.Equals(metadataInputFileInfo.FullName, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("The metadata refers to different file than the one requested");
}
if (!metadataInputFileInfo.Exists)
{
throw new InvalidOperationException("The metadata refers to a file that does not exist");
}
if (metadata.FileLength != metadataInputFileInfo.Length)
{
throw new InvalidOperationException("The metadata's file information differs from the actual file");
}
}
/// <summary>
/// Attempts to load the metadata from an existing file in its canonical location.
/// </summary>
/// <param name="metadataFilePath">The metadata file path.</param>
/// <returns></returns>
public UploadMetadata GetExistingMetadata(string metadataFilePath)
{
//load from file (based on input parameters)
var metadata = UploadMetadata.LoadFrom(metadataFilePath);
metadata.ValidateConsistency();
return(metadata);
}
/// <summary>
/// Updates the segment metadata status.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="newStatus">The new status.</param>
private static void UpdateSegmentMetadataStatus(UploadMetadata metadata, int segmentNumber, SegmentUploadStatus newStatus)
{
metadata.Segments[segmentNumber].Status = newStatus;
try
{
metadata.Save();
}
catch { } //no need to crash the program if were unable to save the metadata; it is what's in memory that's important
}
/// <summary>
/// Uploads the file using the given metadata.
///
/// </summary>
/// <param name="metadata"></param>
private void UploadFile(UploadMetadata metadata)
{
try
{
//we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
_previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
ServicePointManager.DefaultConnectionLimit = Math.Max(this.Parameters.ThreadCount,
ServicePointManager.DefaultConnectionLimit);
//match up the metadata with the information on the server
if (this.Parameters.IsResume)
{
ValidateMetadataForResume(metadata);
}
else
{
ValidateMetadataForFreshUpload(metadata);
}
var segmentProgressTracker = CreateSegmentProgressTracker(metadata);
if (metadata.SegmentCount == 0)
{
// simply create the target stream, overwriting existing streams if they exist
_frontEnd.CreateStream(metadata.TargetStreamPath, true, null, 0);
}
else if (metadata.SegmentCount > 1)
{
//perform the multi-segment upload
var msu = new MultipleSegmentUploader(metadata, this.Parameters.ThreadCount, _frontEnd, _token,
segmentProgressTracker);
msu.UseSegmentBlockBackOffRetryStrategy = this.Parameters.UseSegmentBlockBackOffRetryStrategy;
msu.Upload();
//concatenate the files at the end
ConcatenateSegments(metadata);
}
else
{
//optimization if we only have one segment: upload it directly to the target stream
metadata.Segments[0].Path = metadata.TargetStreamPath;
var ssu = new SingleSegmentUploader(0, metadata, _frontEnd, _token, segmentProgressTracker);
ssu.UseBackOffRetryStrategy = this.Parameters.UseSegmentBlockBackOffRetryStrategy;
ssu.Upload();
}
}
catch (OperationCanceledException)
{
// do nothing since we have already marked everything as failed
}
finally
{
//revert back the default .NET value for max connections to a host to whatever it was before
ServicePointManager.DefaultConnectionLimit = _previousDefaultConnectionLimit;
}
}
/// <summary>
/// Creates a new UploadSegmentMetadata with the given segment number.
/// </summary>
/// <param name="segmentNumber"></param>
/// <param name="metadata"></param>
internal UploadSegmentMetadata(int segmentNumber, UploadMetadata metadata)
{
this.SegmentNumber = segmentNumber;
this.Status = SegmentUploadStatus.Pending;
string ignored;
var targetStreamName = metadata.SplitTargetStreamPathByName(out ignored);
this.Path = string.Format("{0}/{1}.{2}.segment{3}", metadata.SegmentStreamDirectory, targetStreamName, metadata.UploadId, this.SegmentNumber);
this.Offset = this.SegmentNumber * metadata.SegmentLength; // segment number is zero-based
this.Length = CalculateSegmentLength(this.SegmentNumber, metadata);
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="token">The cancellation token to use</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_segmentMetadata = uploadMetadata.Segments[segmentNumber];
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseBackOffRetryStrategy = true;
}
/// <summary>
/// Verifies that the metadata is valid for a fresh upload.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataForFreshUpload(UploadMetadata metadata)
{
ValidateMetadataMatchesLocalFile(metadata);
//verify that the target stream does not already exist (in case we don't want to overwrite)
if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath))
{
throw new InvalidOperationException("Target Stream already exists");
}
}
/// <summary>
/// Creates a new metadata based on the given input parameters, and saves it to its canonical location.
/// </summary>
/// <param name="metadataFilePath">The metadata file path.</param>
/// <returns></returns>
public UploadMetadata CreateNewMetadata(string metadataFilePath)
{
//create metadata
var metadata = new UploadMetadata(metadataFilePath, _parameters, _frontend);
//save the initial version
metadata.Save();
return(metadata);
}
/// <summary>
/// Gets the pending segments to upload.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <returns></returns>
private static Queue <SegmentQueueItem> GetPendingSegmentsToUpload(UploadMetadata metadata)
{
var result = new Queue <SegmentQueueItem>();
foreach (var segment in metadata.Segments.Where(segment => segment.Status == SegmentUploadStatus.Pending))
{
result.Enqueue(new SegmentQueueItem(segment.SegmentNumber, 0));
}
return(result);
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="token">The cancellation token to use.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_maxThreadCount = maxThreadCount;
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseSegmentBlockBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="token">The cancellation token to use</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_segmentMetadata = uploadMetadata.Segments[segmentNumber];
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="token">The cancellation token to use.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_maxThreadCount = maxThreadCount;
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseSegmentBlockBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new UploadSegmentMetadata with the given segment number.
/// </summary>
/// <param name="segmentNumber"></param>
/// <param name="metadata"></param>
internal UploadSegmentMetadata(int segmentNumber, UploadMetadata metadata)
{
this.SegmentNumber = segmentNumber;
this.Status = SegmentUploadStatus.Pending;
string ignored;
var targetStreamName = metadata.SplitTargetStreamPathByName(out ignored);
this.Path = string.Format("{0}/{1}.{2}.segment{3}", metadata.SegmentStreamDirectory, targetStreamName, metadata.UploadId, this.SegmentNumber);
this.Offset = this.SegmentNumber * metadata.SegmentLength; // segment number is zero-based
this.Length = CalculateSegmentLength(this.SegmentNumber, metadata);
}
/// <summary>
/// Aligns segments to match record boundaries (where a record boundary = a new line).
/// If not possible (max record size = 4MB), throws an exception.
/// </summary>
/// <param name="metadata"></param>
private void AlignSegmentsToRecordBoundaries(UploadMetadata metadata)
{
int remainingSegments = 0;
using (var stream = new FileStream(metadata.InputFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
long offset = 0;
for (int i = 0; i < metadata.Segments.Length; i++)
{
var segment = metadata.Segments[i];
//updating segment lengths means that both the offset and the length of the next segment needs to be recalculated, to keep the segment lengths somewhat balanced
long diff = segment.Offset - offset;
segment.Offset = offset;
segment.Length += diff;
if (segment.Offset >= metadata.FileLength)
{
continue;
}
if (segment.SegmentNumber == metadata.Segments.Length - 1)
{
//last segment picks up the slack
segment.Length = metadata.FileLength - segment.Offset;
}
else
{
//figure out how much do we need to adjust the length of the segment so it ends on a record boundary (this can be negative or positive)
int lengthAdjustment = DetermineLengthAdjustment(segment, stream) + 1;
//adjust segment length and offset
segment.Length += lengthAdjustment;
}
offset += segment.Length;
remainingSegments++;
}
}
//since we adjusted the segment lengths, it's possible that the last segment(s) became of zero length; so remove it
var segments = metadata.Segments;
if (remainingSegments < segments.Length)
{
Array.Resize(ref segments, remainingSegments);
metadata.Segments = segments;
metadata.SegmentCount = segments.Length;
}
//NOTE: we are not validating consistency here; this method is called by CreateNewMetadata which calls Save() after this, which validates consistency anyway.
}
/// <summary>
/// Creates a new metadata based on the given input parameters, and saves it to its canonical location.
/// </summary>
/// <returns></returns>
public UploadMetadata CreateNewMetadata(string metadataFilePath)
{
//determine segment count, segment length and Upload Id
//create metadata
var metadata = new UploadMetadata(metadataFilePath, _parameters);
if (!_parameters.IsBinary && metadata.SegmentCount > 1)
{
this.AlignSegmentsToRecordBoundaries(metadata);
}
//save the initial version
metadata.Save();
return(metadata);
}
/// <summary>
/// Creates a new metadata based on the given input parameters, and saves it to its canonical location.
/// </summary>
/// <returns></returns>
public UploadMetadata CreateNewMetadata(string metadataFilePath)
{
//determine segment count, segment length and Upload Id
//create metadata
var metadata = new UploadMetadata(metadataFilePath, _parameters);
if (!_parameters.IsBinary && metadata.SegmentCount > 1)
{
this.AlignSegmentsToRecordBoundaries(metadata);
}
//save the initial version
metadata.Save();
return metadata;
}
/// <summary>
/// Updates the progress to indicate that a file failed
/// </summary>
internal void OnFileUploadThreadAborted(UploadMetadata failedFile)
{
++this.UploadedFileCount;
var previousProgress = _fileProgress.Where(p => p.UploadId.Equals(failedFile.UploadId, StringComparison.InvariantCultureIgnoreCase)).First();
foreach (var segment in previousProgress._segmentProgress)
{
// only fail out segments that haven't been completed.
if (segment.Length != segment.UploadedByteCount)
{
segment.IsFailed = true;
}
previousProgress.SetSegmentProgress(segment);
}
}
/// <summary>
/// Creates the segment progress tracker.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <returns></returns>
private IProgress <SegmentUploadProgress> CreateSegmentProgressTracker(UploadMetadata metadata)
{
if (_progressTracker == null)
{
return(null);
}
var overallProgress = new UploadProgress(metadata);
return(new Progress <SegmentUploadProgress>(
(sup) =>
{
//update the overall progress and report it back
overallProgress.SetSegmentProgress(sup);
_progressTracker.Report(overallProgress);
}));
}
/// <summary>
/// Calculates the length of the segment with given number for a file with given length that is split into the given number of segments.
/// </summary>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="metadata">The metadata for the current upload.</param>
/// <returns></returns>
internal static long CalculateSegmentLength(int segmentNumber, UploadMetadata metadata)
{
if (segmentNumber < 0 || segmentNumber >= metadata.SegmentCount)
{
throw new ArgumentOutOfRangeException("segmentNumber", "Segment Number must be at least zero and less than the total number of segments");
}
if (metadata.FileLength < 0)
{
throw new ArgumentException("fileLength", "Cannot have a negative file length");
}
//verify if the last segment would have a positive value
long lastSegmentLength = metadata.FileLength - (metadata.SegmentCount - 1) * metadata.SegmentLength;
if (lastSegmentLength < 0)
{
throw new ArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)");
}
else if (lastSegmentLength > metadata.SegmentLength)
{
//verify if the given segmentCount and segmentLength combination would produce an even split
if (metadata.FileLength - (metadata.SegmentCount - 1) * (metadata.SegmentLength + 1) > 0)
{
throw new ArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength");
}
}
if (metadata.FileLength == 0)
{
return(0);
}
//all segments except the last one have the same length;
//the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment Length
if (segmentNumber < metadata.SegmentCount - 1)
{
return(metadata.SegmentLength);
}
else
{
return(lastSegmentLength);
}
}
/// <summary>
/// Populates the specified metadata.
/// </summary>
/// <param name="metadata">The metadata.</param>
private void Populate(UploadMetadata metadata)
{
this.TotalFileLength = metadata.FileLength;
this.TotalSegmentCount = metadata.SegmentCount;
this.UploadId = metadata.UploadId;
_segmentProgress = new SegmentUploadProgress[this.TotalSegmentCount];
foreach (var segmentInfo in metadata.Segments)
{
if (segmentInfo.Status == SegmentUploadStatus.Complete)
{
this.UploadedByteCount += segmentInfo.Length;
_segmentProgress[segmentInfo.SegmentNumber] = new SegmentUploadProgress(segmentInfo.SegmentNumber, segmentInfo.Length, segmentInfo.Length, false);
}
else
{
_segmentProgress[segmentInfo.SegmentNumber] = new SegmentUploadProgress(segmentInfo.SegmentNumber, segmentInfo.Length, 0, false);
}
}
}
/// <summary>
/// Uploads the segment.
/// </summary>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="metadata">The metadata.</param>
private void UploadSegment(int segmentNumber, UploadMetadata metadata)
{
//mark the segment as 'InProgress' in the metadata
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.InProgress);
var segmentUploader = new SingleSegmentUploader(segmentNumber, metadata, _frontEnd, _token, _progressTracker);
segmentUploader.UseBackOffRetryStrategy = this.UseSegmentBlockBackOffRetryStrategy;
try
{
segmentUploader.Upload();
//if we reach this point, the upload was successful; mark it as such
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.Complete);
}
catch
{
//something horrible happened, mark the segment as failed and throw the original exception (the caller will handle it)
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.Failed);
throw;
}
}
private void VerifySegmentsAreOnRecordBoundaries(UploadMetadata metadata, string filePath)
{
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
foreach (var segment in metadata.Segments)
{
if (segment.SegmentNumber > 0)
{
//verify that each segment starts with a non-newline and that the 2 previous characters before that offset are newline characters
//2 characters behind: newline
stream.Seek(segment.Offset - 2, SeekOrigin.Begin);
char c1 = (char)stream.ReadByte();
Assert.True(IsNewline(c1), string.Format("Expecting a newline at offset {0}", stream.Position - 1));
//1 character behind: newline
char c2 = (char)stream.ReadByte();
Assert.True(IsNewline(c2), string.Format("Expecting a newline at offset {0}", stream.Position - 1));
//by test design, we never have two consecutive newlines that are the same; we'd always have \r\n, but never \r\r or \r\n
var c3 = (char)stream.ReadByte();
Assert.NotEqual(c2, c3);
}
}
}
}
/// <summary>
/// Constructs a new UploadMetadata from the given parameters.
/// </summary>
/// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param>
/// <param name="uploadParameters">The parameters to use for constructing this metadata.</param>
/// <param name="frontend">The frontend to use when generating per file metadata.</param>
public UploadFolderMetadata(string metadataFilePath, UploadParameters uploadParameters, IFrontEndAdapter frontend)
{
this.MetadataFilePath = metadataFilePath;
this.UploadId = Guid.NewGuid().ToString("N");
this.InputFolderPath = uploadParameters.InputFilePath;
this.TargetStreamFolderPath = uploadParameters.TargetStreamPath.TrimEnd('/');
this.IsRecursive = uploadParameters.IsRecursive;
// get this list of all files in the source directory, depending on if this is recursive or not.
ConcurrentQueue <string> allFiles;
ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>();
Dictionary <string, long> downloadFiles = new Dictionary <string, long>();
if (uploadParameters.IsDownload)
{
foreach (var entry in frontend.ListDirectory(uploadParameters.InputFilePath, uploadParameters.IsRecursive))
{
downloadFiles.Add(entry.Key, entry.Value);
}
allFiles = new ConcurrentQueue <string>(downloadFiles.Keys);
this.TotalFileBytes = downloadFiles.Values.Sum();
}
else
{
allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) :
Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly));
this.TotalFileBytes = GetByteCountFromFileList(allFiles);
}
this.FileCount = allFiles.Count();
this.Files = new UploadMetadata[this.FileCount];
// explicitly set the thread pool start amount to at most 500
int threadCount = Math.Min(this.FileCount, 500);
var threads = new List <Thread>(threadCount);
//start a bunch of new threads that will create the metadata and ensure a protected index.
int currentIndex = 0;
object indexIncrementLock = new object();
for (int i = 0; i < threadCount; i++)
{
var t = new Thread(() => {
string curFile;
while (allFiles.TryDequeue(out curFile))
{
try
{
var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/');
var paramsPerFile = new UploadParameters
(
curFile,
String.Format("{0}{1}{2}", this.TargetStreamFolderPath, uploadParameters.IsDownload ? "\\" : "/", relativeFilePath),
uploadParameters.AccountName,
uploadParameters.PerFileThreadCount,
uploadParameters.ConcurrentFileCount,
uploadParameters.IsOverwrite,
uploadParameters.IsResume,
uploadParameters.IsBinary,
uploadParameters.IsRecursive,
uploadParameters.IsDownload,
uploadParameters.MaxSegementLength,
uploadParameters.LocalMetadataLocation
);
long size = -1;
if (uploadParameters.IsDownload && downloadFiles != null)
{
size = downloadFiles[curFile];
}
var uploadMetadataPath = Path.Combine(uploadParameters.LocalMetadataLocation, string.Format("{0}.upload.xml", Path.GetFileName(curFile)));
var eachFileMetadata = new UploadMetadata(uploadMetadataPath, paramsPerFile, frontend, size);
lock (indexIncrementLock)
{
this.Files[currentIndex] = eachFileMetadata;
currentIndex++;
}
}
catch (Exception e)
{
exceptions.Enqueue(e);
}
}
});
t.Start();
threads.Add(t);
}
foreach (var t in threads)
{
t.Join();
}
if (exceptions.Count > 0)
{
throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray());
}
}
/// <summary>
/// Creates the segment progress tracker.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <returns></returns>
private IProgress<SegmentUploadProgress> CreateSegmentProgressTracker(UploadMetadata metadata)
{
if (_progressTracker == null)
{
return null;
}
var overallProgress = new UploadProgress(metadata);
return new Progress<SegmentUploadProgress>(
(sup) =>
{
//update the overall progress and report it back
overallProgress.SetSegmentProgress(sup);
_progressTracker.Report(overallProgress);
});
}
/// <summary>
/// Concatenates all the segments defined in the metadata into a single stream.
/// </summary>
/// <param name="metadata"></param>
private void ConcatenateSegments(UploadMetadata metadata)
{
string[] inputPaths = new string[metadata.SegmentCount];
//verify if target stream exists
if (_frontEnd.StreamExists(metadata.TargetStreamPath))
{
if (this.Parameters.IsOverwrite)
{
_frontEnd.DeleteStream(metadata.TargetStreamPath);
}
else
{
throw new InvalidOperationException("Target Stream already exists");
}
}
//ensure all input streams exist and are of the expected length
//ensure all segments in the metadata are marked as 'complete'
var exceptions = new List<Exception>();
Parallel.For(
0,
metadata.SegmentCount,
new ParallelOptions() { MaxDegreeOfParallelism = this.Parameters.ThreadCount },
(i) =>
{
try
{
if (metadata.Segments[i].Status != SegmentUploadStatus.Complete)
{
throw new UploadFailedException("Cannot perform 'Concatenate' operation because not all streams are fully uploaded.");
}
var remoteStreamPath = metadata.Segments[i].Path;
var retryCount = 0;
long remoteLength = -1;
while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
remoteLength = _frontEnd.GetStreamLength(remoteStreamPath);
break;
}
catch (Exception e)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
throw new UploadFailedException(
string.Format(
"Cannot perform 'Concatenate' operation due to the following exception retrieving file information: {0}",
e));
}
SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token);
}
}
if (remoteLength != metadata.Segments[i].Length)
{
throw new UploadFailedException(string.Format("Cannot perform 'Concatenate' operation because segment {0} has an incorrect length (expected {1}, actual {2}).", i, metadata.Segments[i].Length, remoteLength));
}
inputPaths[i] = remoteStreamPath;
}
catch (Exception ex)
{
//collect any exceptions, whether we just generated them above or whether they come from the Front End,
exceptions.Add(ex);
}
});
if (exceptions.Count > 0)
{
throw new AggregateException("At least one concatenate test failed", exceptions.ToArray());
}
//issue the command
_frontEnd.Concatenate(metadata.TargetStreamPath, inputPaths);
}
public void UploadSegmentMetadata_CalculateParticularSegmentLength()
{
//verify bad inputs
Assert.Throws<ArgumentOutOfRangeException>(
() => { UploadSegmentMetadata.CalculateSegmentLength(-1, new UploadMetadata() { FileLength = 10, SegmentCount = 5, SegmentLength = 2 }); });
Assert.Throws<ArgumentOutOfRangeException>(
() => { UploadSegmentMetadata.CalculateSegmentLength(100, new UploadMetadata() { FileLength = 10, SegmentCount = 5, SegmentLength = 2 }); });
Assert.Throws<ArgumentException>(
() => { UploadSegmentMetadata.CalculateSegmentLength(1, new UploadMetadata() { FileLength = -10, SegmentCount = 5, SegmentLength = 2 }); });
Assert.Throws<ArgumentException>(
() => { UploadSegmentMetadata.CalculateSegmentLength(1, new UploadMetadata() { FileLength = 100, SegmentCount = 2, SegmentLength = 2 }); });
Assert.Throws<ArgumentException>(
() => { UploadSegmentMetadata.CalculateSegmentLength(1, new UploadMetadata() { FileLength = 100, SegmentCount = 5, SegmentLength = 26 }); });
//test various scenarios with a fixed file length, and varying the segment count from 1 to the FileLength
int FileLength = 16 * (int)Math.Pow(2, 20);//16MB
for (int segmentCount = 1; segmentCount <= FileLength; segmentCount += 1024)
{
long typicalSegmentLength = UploadSegmentMetadata.CalculateSegmentLength(FileLength, segmentCount);
var uploadMetadata = new UploadMetadata(){FileLength=FileLength,SegmentCount=segmentCount,SegmentLength=typicalSegmentLength};
long firstSegmentLength = UploadSegmentMetadata.CalculateSegmentLength(0, uploadMetadata);
long lastSegmentLength = UploadSegmentMetadata.CalculateSegmentLength(segmentCount - 1, uploadMetadata);
Assert.Equal(typicalSegmentLength, firstSegmentLength);
if (segmentCount == 1)
{
Assert.Equal(firstSegmentLength, lastSegmentLength);
}
long reconstructedFileLength = typicalSegmentLength * (segmentCount - 1) + lastSegmentLength;
Assert.Equal(FileLength, reconstructedFileLength);
}
}
/// <summary>
/// Calculates the length of the segment with given number for a file with given length that is split into the given number of segments.
/// </summary>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="metadata">The metadata for the current upload.</param>
/// <returns></returns>
internal static long CalculateSegmentLength(int segmentNumber, UploadMetadata metadata)
{
if (segmentNumber < 0 || segmentNumber >= metadata.SegmentCount)
{
throw new ArgumentOutOfRangeException("segmentNumber", "Segment Number must be at least zero and less than the total number of segments");
}
if (metadata.FileLength < 0)
{
throw new ArgumentException("fileLength", "Cannot have a negative file length");
}
//verify if the last segment would have a positive value
long lastSegmentLength = metadata.FileLength - (metadata.SegmentCount - 1) * metadata.SegmentLength;
if (lastSegmentLength < 0)
{
throw new ArgumentException("The given values for segmentCount and segmentLength cannot possibly be used to split a file with the given fileLength (the last segment would have a negative length)");
}
else if (lastSegmentLength > metadata.SegmentLength)
{
//verify if the given segmentCount and segmentLength combination would produce an even split
if (metadata.FileLength - (metadata.SegmentCount - 1) * (metadata.SegmentLength + 1) > 0)
{
throw new ArgumentException("The given values for segmentCount and segmentLength would not produce an even split of a file with given fileLength");
}
}
if (metadata.FileLength == 0)
{
return 0;
}
//all segments except the last one have the same length;
//the last one only has the 'full' length if by some miracle the file length is a perfect multiple of the Segment Length
if (segmentNumber < metadata.SegmentCount - 1)
{
return metadata.SegmentLength;
}
else
{
return lastSegmentLength;
}
}
/// <summary>
/// Gets the pending segments to upload.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <returns></returns>
private static Queue<SegmentQueueItem> GetPendingSegmentsToUpload(UploadMetadata metadata)
{
var result = new Queue<SegmentQueueItem>();
foreach (var segment in metadata.Segments.Where(segment => segment.Status == SegmentUploadStatus.Pending))
{
result.Enqueue(new SegmentQueueItem(segment.SegmentNumber, 0));
}
return result;
}
/// <summary>
/// Concatenates all the segments defined in the metadata into a single stream.
/// </summary>
/// <param name="metadata"></param>
private void ConcatenateSegments(UploadMetadata metadata)
{
string[] inputPaths = new string[metadata.SegmentCount];
//verify if target stream exists
if (_frontEnd.StreamExists(metadata.TargetStreamPath))
{
if (this.Parameters.IsOverwrite)
{
_frontEnd.DeleteStream(metadata.TargetStreamPath);
}
else
{
throw new InvalidOperationException("Target Stream already exists");
}
}
//ensure all input streams exist and are of the expected length
//ensure all segments in the metadata are marked as 'complete'
var exceptions = new List <Exception>();
Parallel.For(
0,
metadata.SegmentCount,
new ParallelOptions()
{
MaxDegreeOfParallelism = this.Parameters.ThreadCount
},
(i) =>
{
try
{
if (metadata.Segments[i].Status != SegmentUploadStatus.Complete)
{
throw new UploadFailedException("Cannot perform 'Concatenate' operation because not all streams are fully uploaded.");
}
var remoteStreamPath = metadata.Segments[i].Path;
var retryCount = 0;
long remoteLength = -1;
while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
remoteLength = _frontEnd.GetStreamLength(remoteStreamPath);
break;
}
catch (Exception e)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
throw new UploadFailedException(
string.Format(
"Cannot perform 'Concatenate' operation due to the following exception retrieving file information: {0}",
e));
}
SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token);
}
}
if (remoteLength != metadata.Segments[i].Length)
{
throw new UploadFailedException(string.Format("Cannot perform 'Concatenate' operation because segment {0} has an incorrect length (expected {1}, actual {2}).", i, metadata.Segments[i].Length, remoteLength));
}
inputPaths[i] = remoteStreamPath;
}
catch (Exception ex)
{
//collect any exceptions, whether we just generated them above or whether they come from the Front End,
exceptions.Add(ex);
}
});
if (exceptions.Count > 0)
{
throw new AggregateException("At least one concatenate test failed", exceptions.ToArray());
}
//issue the command
_frontEnd.Concatenate(metadata.TargetStreamPath, inputPaths);
}
private void VerifyTargetStreamIsComplete(UploadSegmentMetadata segmentMetadata, UploadMetadata metadata, InMemoryFrontEnd frontEnd)
{
Assert.Equal(SegmentUploadStatus.Complete, segmentMetadata.Status);
Assert.True(frontEnd.StreamExists(segmentMetadata.Path), string.Format("Segment {0} was not uploaded", segmentMetadata.SegmentNumber));
Assert.Equal(segmentMetadata.Length, frontEnd.GetStreamLength(segmentMetadata.Path));
var actualContents = frontEnd.GetStreamContents(segmentMetadata.Path);
var expectedContents = GetExpectedContents(segmentMetadata, metadata);
AssertExtensions.AreEqual(expectedContents, actualContents, "Segment {0} has unexpected contents", segmentMetadata.SegmentNumber);
}
/// <summary>
/// Uploads the segment.
/// </summary>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="metadata">The metadata.</param>
private void UploadSegment(int segmentNumber, UploadMetadata metadata)
{
//mark the segment as 'InProgress' in the metadata
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.InProgress);
var segmentUploader = new SingleSegmentUploader(segmentNumber, metadata, _frontEnd, _token, _progressTracker);
segmentUploader.UseBackOffRetryStrategy = this.UseSegmentBlockBackOffRetryStrategy;
try
{
segmentUploader.Upload();
//if we reach this point, the upload was successful; mark it as such
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.Complete);
}
catch
{
//something horrible happened, mark the segment as failed and throw the original exception (the caller will handle it)
UpdateSegmentMetadataStatus(metadata, segmentNumber, SegmentUploadStatus.Failed);
throw;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="UploadProgress" /> class.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <param name="progressTracker">The progress tracker.</param>
internal UploadProgress(UploadMetadata metadata, IProgress <UploadProgress> progressTracker = null)
{
_progressTracker = progressTracker;
Populate(metadata);
}
private UploadMetadata CreateMetadata(int segmentCount)
{
var path = Path.GetTempFileName();
var metadata = new UploadMetadata()
{
MetadataFilePath = path,
InputFilePath = _smallFilePath,
FileLength = _smallFileContents.Length,
SegmentCount = segmentCount,
SegmentLength = UploadSegmentMetadata.CalculateSegmentLength(_smallFileContents.Length, segmentCount),
Segments = new UploadSegmentMetadata[segmentCount],
TargetStreamPath = "abc",
UploadId = "123",
IsBinary = true
};
long offset = 0;
for (int i = 0; i < segmentCount; i++)
{
long length = UploadSegmentMetadata.CalculateSegmentLength(i, metadata);
metadata.Segments[i] = new UploadSegmentMetadata()
{
SegmentNumber = i,
Offset = offset,
Status = SegmentUploadStatus.Pending,
Length = length,
Path = string.Format("{0}.{1}.segment{2}", metadata.TargetStreamPath, metadata.UploadId, i)
};
offset += length;
}
return metadata;
}
/// <summary>
/// Updates the segment metadata status.
/// </summary>
/// <param name="metadata">The metadata.</param>
/// <param name="segmentNumber">The segment number.</param>
/// <param name="newStatus">The new status.</param>
private static void UpdateSegmentMetadataStatus(UploadMetadata metadata, int segmentNumber, SegmentUploadStatus newStatus)
{
metadata.Segments[segmentNumber].Status = newStatus;
try
{
metadata.Save();
}
catch { } //no need to crash the program if were unable to save the metadata; it is what's in memory that's important
}
private byte[] GetExpectedContents(UploadSegmentMetadata segment, UploadMetadata metadata)
{
byte[] result = new byte[segment.Length];
Array.Copy(_smallFileContents, segment.SegmentNumber * metadata.SegmentLength, result, 0, segment.Length);
return result;
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) :
this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker)
{
}
private void VerifyTargetStreamsAreComplete(UploadMetadata metadata, InMemoryFrontEnd fe)
{
foreach (var segment in metadata.Segments)
{
VerifyTargetStreamIsComplete(segment, metadata, fe);
}
}
/// <summary>
/// Validates that the metadata is valid for a resume operation, and also updates the internal Segment States to match what the Server looks like.
/// If any changes are made, the metadata will be saved to its canonical location.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataForResume(UploadMetadata metadata)
{
ValidateMetadataMatchesLocalFile(metadata);
//verify that the target stream does not already exist (in case we don't want to overwrite)
if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath))
{
throw new InvalidOperationException("Target Stream already exists");
}
//make sure we don't upload part of the file as binary, while the rest is non-binary (that's just asking for trouble)
if (this.Parameters.IsBinary != metadata.IsBinary)
{
throw new InvalidOperationException(
string.Format(
"Existing metadata was created for a {0}binary file while the current parameters requested a {1}binary upload.",
metadata.IsBinary ? string.Empty : "non-",
this.Parameters.IsBinary ? string.Empty : "non-"));
}
//see what files(segments) already exist - update metadata accordingly (only for segments that are missing from server; if it's on the server but not in metadata, reupload)
foreach (var segment in metadata.Segments)
{
if (segment.Status == SegmentUploadStatus.Complete)
{
var retryCount = 0;
while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
//verify that the stream exists and that the length is as expected
if (!_frontEnd.StreamExists(segment.Path))
{
// this segment was marked as completed, but no target stream exists; it needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
else
{
var remoteLength = _frontEnd.GetStreamLength(segment.Path);
if (remoteLength != segment.Length)
{
//the target stream has a different length than the input segment, which implies they are inconsistent; it needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
}
break;
}
catch (Exception e)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
throw new UploadFailedException(
string.Format(
"Cannot validate metadata in order to resume due to the following exception retrieving file information: {0}",
e));
}
SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token);
}
}
}
else
{
//anything which is not in 'Completed' status needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
}
metadata.Save();
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress <SegmentUploadProgress> progressTracker = null) :
this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Verifies that the metadata is valid for a fresh upload.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataForFreshUpload(UploadMetadata metadata)
{
ValidateMetadataMatchesLocalFile(metadata);
//verify that the target stream does not already exist (in case we don't want to overwrite)
if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath))
{
throw new InvalidOperationException("Target Stream already exists");
}
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentUploadProgress> progressTracker = null) :
this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Verifies that the metadata is consistent with the local file information.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataMatchesLocalFile(UploadMetadata metadata)
{
if (metadata.TargetStreamPath.Trim() != this.Parameters.TargetStreamPath.Trim())
{
throw new InvalidOperationException("Metadata points to a different target stream than the input parameters");
}
//verify that it matches against local file (size, name)
var metadataInputFileInfo = new FileInfo(metadata.InputFilePath);
var paramInputFileInfo = new FileInfo(this.Parameters.InputFilePath);
if (!paramInputFileInfo.FullName.Equals(metadataInputFileInfo.FullName, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("The metadata refers to different file than the one requested");
}
if (!metadataInputFileInfo.Exists)
{
throw new InvalidOperationException("The metadata refers to a file that does not exist");
}
if (metadata.FileLength != metadataInputFileInfo.Length)
{
throw new InvalidOperationException("The metadata's file information differs from the actual file");
}
}
/// <summary>
/// Uploads the file using the given metadata.
///
/// </summary>
/// <param name="metadata"></param>
private void UploadFile(UploadMetadata metadata)
{
try
{
//we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
_previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
ServicePointManager.DefaultConnectionLimit = Math.Max(this.Parameters.ThreadCount,
ServicePointManager.DefaultConnectionLimit);
//match up the metadata with the information on the server
if (this.Parameters.IsResume)
{
ValidateMetadataForResume(metadata);
}
else
{
ValidateMetadataForFreshUpload(metadata);
}
var segmentProgressTracker = CreateSegmentProgressTracker(metadata);
if (metadata.SegmentCount == 0)
{
// simply create the target stream, overwriting existing streams if they exist
_frontEnd.CreateStream(metadata.TargetStreamPath, true, null, 0);
}
else if (metadata.SegmentCount > 1)
{
//perform the multi-segment upload
var msu = new MultipleSegmentUploader(metadata, this.Parameters.ThreadCount, _frontEnd, _token,
segmentProgressTracker);
msu.UseSegmentBlockBackOffRetryStrategy = this.Parameters.UseSegmentBlockBackOffRetryStrategy;
msu.Upload();
//concatenate the files at the end
ConcatenateSegments(metadata);
}
else
{
//optimization if we only have one segment: upload it directly to the target stream
metadata.Segments[0].Path = metadata.TargetStreamPath;
var ssu = new SingleSegmentUploader(0, metadata, _frontEnd, _token, segmentProgressTracker);
ssu.UseBackOffRetryStrategy = this.Parameters.UseSegmentBlockBackOffRetryStrategy;
ssu.Upload();
}
}
catch (OperationCanceledException)
{
// do nothing since we have already marked everything as failed
}
finally
{
//revert back the default .NET value for max connections to a host to whatever it was before
ServicePointManager.DefaultConnectionLimit = _previousDefaultConnectionLimit;
}
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) :
this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Aligns segments to match record boundaries (where a record boundary = a new line).
/// If not possible (max record size = 4MB), throws an exception.
/// </summary>
/// <param name="metadata"></param>
private void AlignSegmentsToRecordBoundaries(UploadMetadata metadata)
{
int remainingSegments = 0;
using (var stream = new FileStream(metadata.InputFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
long offset = 0;
for (int i = 0; i < metadata.Segments.Length; i++)
{
var segment = metadata.Segments[i];
//updating segment lengths means that both the offset and the length of the next segment needs to be recalculated, to keep the segment lengths somewhat balanced
long diff = segment.Offset - offset;
segment.Offset = offset;
segment.Length += diff;
if (segment.Offset >= metadata.FileLength)
{
continue;
}
if (segment.SegmentNumber == metadata.Segments.Length - 1)
{
//last segment picks up the slack
segment.Length = metadata.FileLength - segment.Offset;
}
else
{
//figure out how much do we need to adjust the length of the segment so it ends on a record boundary (this can be negative or positive)
int lengthAdjustment = DetermineLengthAdjustment(segment, stream) + 1;
//adjust segment length and offset
segment.Length += lengthAdjustment;
}
offset += segment.Length;
remainingSegments++;
}
}
//since we adjusted the segment lengths, it's possible that the last segment(s) became of zero length; so remove it
var segments = metadata.Segments;
if (remainingSegments < segments.Length)
{
Array.Resize(ref segments, remainingSegments);
metadata.Segments = segments;
metadata.SegmentCount = segments.Length;
}
//NOTE: we are not validating consistency here; this method is called by CreateNewMetadata which calls Save() after this, which validates consistency anyway.
}
/// <summary>
/// Validates that the metadata is valid for a resume operation, and also updates the internal Segment States to match what the Server looks like.
/// If any changes are made, the metadata will be saved to its canonical location.
/// </summary>
/// <param name="metadata"></param>
private void ValidateMetadataForResume(UploadMetadata metadata)
{
ValidateMetadataMatchesLocalFile(metadata);
//verify that the target stream does not already exist (in case we don't want to overwrite)
if (!this.Parameters.IsOverwrite && _frontEnd.StreamExists(metadata.TargetStreamPath))
{
throw new InvalidOperationException("Target Stream already exists");
}
//make sure we don't upload part of the file as binary, while the rest is non-binary (that's just asking for trouble)
if (this.Parameters.IsBinary != metadata.IsBinary)
{
throw new InvalidOperationException(
string.Format(
"Existing metadata was created for a {0}binary file while the current parameters requested a {1}binary upload.",
metadata.IsBinary ? string.Empty : "non-",
this.Parameters.IsBinary ? string.Empty : "non-"));
}
//see what files(segments) already exist - update metadata accordingly (only for segments that are missing from server; if it's on the server but not in metadata, reupload)
foreach (var segment in metadata.Segments)
{
if (segment.Status == SegmentUploadStatus.Complete)
{
var retryCount = 0;
while (retryCount < SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
_token.ThrowIfCancellationRequested();
retryCount++;
try
{
//verify that the stream exists and that the length is as expected
if (!_frontEnd.StreamExists(segment.Path))
{
// this segment was marked as completed, but no target stream exists; it needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
else
{
var remoteLength = _frontEnd.GetStreamLength(segment.Path);
if (remoteLength != segment.Length)
{
//the target stream has a different length than the input segment, which implies they are inconsistent; it needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
}
break;
}
catch (Exception e)
{
_token.ThrowIfCancellationRequested();
if (retryCount >= SingleSegmentUploader.MaxBufferUploadAttemptCount)
{
throw new UploadFailedException(
string.Format(
"Cannot validate metadata in order to resume due to the following exception retrieving file information: {0}",
e));
}
SingleSegmentUploader.WaitForRetry(retryCount, Parameters.UseSegmentBlockBackOffRetryStrategy, _token);
}
}
}
else
{
//anything which is not in 'Completed' status needs to be reuploaded
segment.Status = SegmentUploadStatus.Pending;
}
}
metadata.Save();
}
