/// <summary>
/// Constructor with base front end.
/// </summary>
/// <param name="baseAdapter">The front end.</param>
public MockableFrontEnd(IFrontEndAdapter baseAdapter)
{
this.AppendToStreamImplementation = baseAdapter.AppendToStream;
this.ConcatenateImplementation = baseAdapter.Concatenate;
this.CreateStreamImplementation = baseAdapter.CreateStream;
this.DeleteStreamImplementation = baseAdapter.DeleteStream;
this.GetStreamLengthImplementation = baseAdapter.GetStreamLength;
this.StreamExistsImplementation = baseAdapter.StreamExists;
}
/// <summary>
/// Constructor with base front end.
/// </summary>
/// <param name="baseAdapter">The front end.</param>
public MockableFrontEnd(IFrontEndAdapter baseAdapter)
{
this.AppendToStreamImplementation = baseAdapter.AppendToStream;
this.ConcatenateImplementation = baseAdapter.Concatenate;
this.CreateStreamImplementation = baseAdapter.CreateStream;
this.DeleteStreamImplementation = baseAdapter.DeleteStream;
this.GetStreamLengthImplementation = baseAdapter.GetStreamLength;
this.StreamExistsImplementation = baseAdapter.StreamExists;
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="token">The cancellation token to use.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_maxThreadCount = maxThreadCount;
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseSegmentBlockBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="token">The cancellation token to use</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_segmentMetadata = uploadMetadata.Segments[segmentNumber];
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="token">The cancellation token to use</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_segmentMetadata = uploadMetadata.Segments[segmentNumber];
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="token">The cancellation token to use.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null)
{
_metadata = uploadMetadata;
_maxThreadCount = maxThreadCount;
_frontEnd = frontEnd;
_progressTracker = progressTracker;
_token = token;
this.UseSegmentBlockBackOffRetryStrategy = true;
}
/// <summary>
/// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload.
/// </summary>
/// <param name="uploadParameters">The Upload Parameters to use.</param>
/// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param>
/// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param>
public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<UploadProgress> progressTracker = null )
{
this.Parameters = uploadParameters;
_frontEnd = frontEnd;
//ensure that input parameters are correct
ValidateParameters();
_metadataFilePath = GetCanonicalMetadataFilePath();
_progressTracker = progressTracker;
_token = token;
}
/// <summary>
/// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload.
/// </summary>
/// <param name="uploadParameters">The Upload Parameters to use.</param>
/// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param>
/// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param>
public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <UploadProgress> progressTracker = null)
{
this.Parameters = uploadParameters;
_frontEnd = frontEnd;
//ensure that input parameters are correct
ValidateParameters();
_metadataFilePath = GetCanonicalMetadataFilePath();
_progressTracker = progressTracker;
_token = token;
}
/// <summary>
/// Constructor with base front end.
/// </summary>
/// <param name="baseAdapter">The front end.</param>
public MockableFrontEnd(IFrontEndAdapter baseAdapter)
{
this.AppendToStreamImplementation = baseAdapter.AppendToStream;
this.ConcatenateImplementation = baseAdapter.Concatenate;
this.CreateStreamImplementation = baseAdapter.CreateStream;
this.DeleteStreamImplementation = baseAdapter.DeleteStream;
this.GetStreamLengthImplementation = baseAdapter.GetStreamLength;
this.StreamExistsImplementation = baseAdapter.StreamExists;
this.ReadStreamImplementation = baseAdapter.ReadStream;
this.IsDirectoryImplementation = baseAdapter.IsDirectory;
this.ListDirectoryImplementation = baseAdapter.ListDirectory;
BaseAdapter = baseAdapter as InMemoryFrontEnd;
}
/// <summary>
/// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload.
/// </summary>
/// <param name="uploadParameters">The Upload Parameters to use.</param>
/// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param>
/// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param>
public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, IProgress <UploadProgress> progressTracker = null) :
this(uploadParameters, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) :
this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Creates a new instance of the TransferMetadataGenerator with the given parameters and the given maximum append length.
/// </summary>
/// <param name="parameters">The parameters.</param>
/// <param name="frontend">The frontend to use when generating new metadata.</param>
public TransferMetadataGenerator(TransferParameters parameters, IFrontEndAdapter frontend)
{
_parameters = parameters;
_frontend = frontend;
}
/// <summary>
/// Creates a new downloader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="downloadMetadata">The metadata for the entire download.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) :
this(segmentNumber, downloadMetadata, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentUploadProgress> progressTracker = null) :
this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Constructs a new TransferMetadata from the given parameters.
/// </summary>
/// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param>
/// <param name="transferParameters">The parameters to use for constructing this metadata.</param>
/// <param name="frontEnd">The front end. This is used only in the constructor for determining file length</param>
internal TransferMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontEnd, long fileSize = -1)
{
this.MetadataFilePath = metadataFilePath;
this.TransferId = Guid.NewGuid().ToString("N");
this.InputFilePath = transferParameters.InputFilePath;
this.TargetStreamPath = transferParameters.TargetStreamPath;
this.IsDownload = transferParameters.IsDownload;
this.SegmentStreamDirectory = GetSegmentStreamDirectory();
this.IsBinary = transferParameters.IsBinary;
this.FileLength = fileSize < 0 ? frontEnd.GetStreamLength(transferParameters.InputFilePath, !IsDownload) : fileSize;
this.EncodingCodePage = transferParameters.FileEncoding.CodePage;
// we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic.
// this protects us against agressive increase of thread count resulting in far more segments than
// is reasonable for a given file size. We also ensure that each segment is at least 256mb in size.
// This is the size that ensures we have the optimal storage creation in the store.
var preliminarySegmentCount = (int)Math.Ceiling((double)this.FileLength / transferParameters.MaxSegementLength);
this.SegmentCount = Math.Min(preliminarySegmentCount, TransferSegmentMetadata.CalculateSegmentCount(this.FileLength));
this.SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(this.FileLength, this.SegmentCount);
this.Segments = new TransferSegmentMetadata[this.SegmentCount];
for (int i = 0; i < this.SegmentCount; i++)
{
this.Segments[i] = new TransferSegmentMetadata(i, this);
}
if (!transferParameters.IsBinary && this.SegmentCount > 1 && !this.IsDownload)
{
this.AlignSegmentsToRecordBoundaries();
// ensure that nothing strange happened during alignment
this.ValidateConsistency();
}
// initialize the status to pending, since it is not yet done.
this.Status = SegmentTransferStatus.Pending;
}
/// <summary>
/// Constructs a new TransferFolderMetadata object from the given parameters.
/// </summary>
/// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param>
/// <param name="transferParameters">The parameters to use for constructing this metadata.</param>
/// <param name="frontend">The frontend to use when generating per file metadata.</param>
public TransferFolderMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontend)
{
this.MetadataFilePath = metadataFilePath;
this.TransferId = Guid.NewGuid().ToString("N");
this.InputFolderPath = transferParameters.InputFilePath;
this.TargetStreamFolderPath = transferParameters.TargetStreamPath.TrimEnd('/');
this.IsRecursive = transferParameters.IsRecursive;
// get this list of all files in the source directory, depending on if this is recursive or not.
ConcurrentQueue <string> allFiles;
ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>();
Dictionary <string, long> downloadFiles = new Dictionary <string, long>();
if (transferParameters.IsDownload)
{
foreach (var entry in frontend.ListDirectory(transferParameters.InputFilePath, transferParameters.IsRecursive))
{
downloadFiles.Add(entry.Key, entry.Value);
}
allFiles = new ConcurrentQueue <string>(downloadFiles.Keys);
this.TotalFileBytes = downloadFiles.Values.Sum();
}
else
{
allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) :
Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly));
this.TotalFileBytes = GetByteCountFromFileList(allFiles);
}
this.FileCount = allFiles.Count();
this.Files = new TransferMetadata[this.FileCount];
// explicitly set the thread pool start amount to at most 500
int threadCount = Math.Min(this.FileCount, 500);
var threads = new List <Thread>(threadCount);
//start a bunch of new threads that will create the metadata and ensure a protected index.
int currentIndex = 0;
object indexIncrementLock = new object();
for (int i = 0; i < threadCount; i++)
{
var t = new Thread(() => {
string curFile;
while (allFiles.TryDequeue(out curFile))
{
try
{
var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/');
var paramsPerFile = new TransferParameters
(
curFile,
String.Format("{0}{1}{2}", this.TargetStreamFolderPath, transferParameters.IsDownload ? "\\" : "/", relativeFilePath),
transferParameters.AccountName,
transferParameters.PerFileThreadCount,
transferParameters.ConcurrentFileCount,
transferParameters.IsOverwrite,
transferParameters.IsResume,
transferParameters.IsBinary,
transferParameters.IsRecursive,
transferParameters.IsDownload,
transferParameters.MaxSegementLength,
transferParameters.LocalMetadataLocation
);
long size = -1;
if (transferParameters.IsDownload && downloadFiles != null)
{
size = downloadFiles[curFile];
}
var transferMetadataPath = Path.Combine(transferParameters.LocalMetadataLocation, string.Format("{0}.transfer.xml", Path.GetFileName(curFile)));
var eachFileMetadata = new TransferMetadata(transferMetadataPath, paramsPerFile, frontend, size);
lock (indexIncrementLock)
{
this.Files[currentIndex] = eachFileMetadata;
currentIndex++;
}
}
catch (Exception e)
{
exceptions.Enqueue(e);
}
}
});
t.Start();
threads.Add(t);
}
foreach (var t in threads)
{
t.Join();
}
if (exceptions.Count > 0)
{
throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray());
}
}
/// <summary>
/// Creates a new MultipleSegmentUploader.
/// </summary>
/// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param>
/// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param>
/// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param>
/// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param>
public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) :
this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload.
/// </summary>
/// <param name="uploadParameters">The Upload Parameters to use.</param>
/// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param>
/// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param>
public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, IProgress<UploadProgress> progressTracker = null) :
this(uploadParameters, frontEnd, CancellationToken.None, progressTracker)
{
}
/// <summary>
/// Creates a new instance of the UploadMetadataGenerator with the given parameters and the given maximum append length.
/// </summary>
/// <param name="parameters">The parameters.</param>
/// <param name="frontend">The frontend to use when generating new metadata.</param>
public UploadMetadataGenerator(UploadParameters parameters, IFrontEndAdapter frontend)
{
_parameters = parameters;
_frontend = frontend;
}
/// <summary>
/// Creates a new uploader for a single segment.
/// </summary>
/// <param name="segmentNumber">The sequence number of the segment.</param>
/// <param name="uploadMetadata">The metadata for the entire upload.</param>
/// <param name="frontEnd">A pointer to the front end.</param>
/// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param>
public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) :
this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker)
{
}
