/// <summary> /// Constructor with base front end. /// </summary> /// <param name="baseAdapter">The front end.</param> public MockableFrontEnd(IFrontEndAdapter baseAdapter) { this.AppendToStreamImplementation = baseAdapter.AppendToStream; this.ConcatenateImplementation = baseAdapter.Concatenate; this.CreateStreamImplementation = baseAdapter.CreateStream; this.DeleteStreamImplementation = baseAdapter.DeleteStream; this.GetStreamLengthImplementation = baseAdapter.GetStreamLength; this.StreamExistsImplementation = baseAdapter.StreamExists; }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="token">The cancellation token to use.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _maxThreadCount = maxThreadCount; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseSegmentBlockBackOffRetryStrategy = true; }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="token">The cancellation token to use</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _segmentMetadata = uploadMetadata.Segments[segmentNumber]; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseBackOffRetryStrategy = true; }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="token">The cancellation token to use</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<SegmentUploadProgress> progressTracker = null) { _metadata = uploadMetadata; _segmentMetadata = uploadMetadata.Segments[segmentNumber]; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseBackOffRetryStrategy = true; }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="token">The cancellation token to use.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <SegmentTransferProgress> progressTracker = null) { _metadata = uploadMetadata; _maxThreadCount = maxThreadCount; _frontEnd = frontEnd; _progressTracker = progressTracker; _token = token; this.UseSegmentBlockBackOffRetryStrategy = true; }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<UploadProgress> progressTracker = null ) { this.Parameters = uploadParameters; _frontEnd = frontEnd; //ensure that input parameters are correct ValidateParameters(); _metadataFilePath = GetCanonicalMetadataFilePath(); _progressTracker = progressTracker; _token = token; }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <UploadProgress> progressTracker = null) { this.Parameters = uploadParameters; _frontEnd = frontEnd; //ensure that input parameters are correct ValidateParameters(); _metadataFilePath = GetCanonicalMetadataFilePath(); _progressTracker = progressTracker; _token = token; }
/// <summary> /// Constructor with base front end. /// </summary> /// <param name="baseAdapter">The front end.</param> public MockableFrontEnd(IFrontEndAdapter baseAdapter) { this.AppendToStreamImplementation = baseAdapter.AppendToStream; this.ConcatenateImplementation = baseAdapter.Concatenate; this.CreateStreamImplementation = baseAdapter.CreateStream; this.DeleteStreamImplementation = baseAdapter.DeleteStream; this.GetStreamLengthImplementation = baseAdapter.GetStreamLength; this.StreamExistsImplementation = baseAdapter.StreamExists; this.ReadStreamImplementation = baseAdapter.ReadStream; this.IsDirectoryImplementation = baseAdapter.IsDirectory; this.ListDirectoryImplementation = baseAdapter.ListDirectory; BaseAdapter = baseAdapter as InMemoryFrontEnd; }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, IProgress <UploadProgress> progressTracker = null) : this(uploadParameters, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(TransferMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) : this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new instance of the TransferMetadataGenerator with the given parameters and the given maximum append length. /// </summary> /// <param name="parameters">The parameters.</param> /// <param name="frontend">The frontend to use when generating new metadata.</param> public TransferMetadataGenerator(TransferParameters parameters, IFrontEndAdapter frontend) { _parameters = parameters; _frontend = frontend; }
/// <summary> /// Creates a new downloader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="downloadMetadata">The metadata for the entire download.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentDownloader(int segmentNumber, TransferMetadata downloadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentTransferProgress> progressTracker = null) : this(segmentNumber, downloadMetadata, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress <SegmentUploadProgress> progressTracker = null) : this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Constructs a new TransferMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="transferParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontEnd">The front end. This is used only in the constructor for determining file length</param> internal TransferMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontEnd, long fileSize = -1) { this.MetadataFilePath = metadataFilePath; this.TransferId = Guid.NewGuid().ToString("N"); this.InputFilePath = transferParameters.InputFilePath; this.TargetStreamPath = transferParameters.TargetStreamPath; this.IsDownload = transferParameters.IsDownload; this.SegmentStreamDirectory = GetSegmentStreamDirectory(); this.IsBinary = transferParameters.IsBinary; this.FileLength = fileSize < 0 ? frontEnd.GetStreamLength(transferParameters.InputFilePath, !IsDownload) : fileSize; this.EncodingCodePage = transferParameters.FileEncoding.CodePage; // we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic. // this protects us against agressive increase of thread count resulting in far more segments than // is reasonable for a given file size. We also ensure that each segment is at least 256mb in size. // This is the size that ensures we have the optimal storage creation in the store. var preliminarySegmentCount = (int)Math.Ceiling((double)this.FileLength / transferParameters.MaxSegementLength); this.SegmentCount = Math.Min(preliminarySegmentCount, TransferSegmentMetadata.CalculateSegmentCount(this.FileLength)); this.SegmentLength = TransferSegmentMetadata.CalculateSegmentLength(this.FileLength, this.SegmentCount); this.Segments = new TransferSegmentMetadata[this.SegmentCount]; for (int i = 0; i < this.SegmentCount; i++) { this.Segments[i] = new TransferSegmentMetadata(i, this); } if (!transferParameters.IsBinary && this.SegmentCount > 1 && !this.IsDownload) { this.AlignSegmentsToRecordBoundaries(); // ensure that nothing strange happened during alignment this.ValidateConsistency(); } // initialize the status to pending, since it is not yet done. this.Status = SegmentTransferStatus.Pending; }
/// <summary> /// Constructs a new TransferFolderMetadata object from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="transferParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontend">The frontend to use when generating per file metadata.</param> public TransferFolderMetadata(string metadataFilePath, TransferParameters transferParameters, IFrontEndAdapter frontend) { this.MetadataFilePath = metadataFilePath; this.TransferId = Guid.NewGuid().ToString("N"); this.InputFolderPath = transferParameters.InputFilePath; this.TargetStreamFolderPath = transferParameters.TargetStreamPath.TrimEnd('/'); this.IsRecursive = transferParameters.IsRecursive; // get this list of all files in the source directory, depending on if this is recursive or not. ConcurrentQueue <string> allFiles; ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>(); Dictionary <string, long> downloadFiles = new Dictionary <string, long>(); if (transferParameters.IsDownload) { foreach (var entry in frontend.ListDirectory(transferParameters.InputFilePath, transferParameters.IsRecursive)) { downloadFiles.Add(entry.Key, entry.Value); } allFiles = new ConcurrentQueue <string>(downloadFiles.Keys); this.TotalFileBytes = downloadFiles.Values.Sum(); } else { allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) : Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly)); this.TotalFileBytes = GetByteCountFromFileList(allFiles); } this.FileCount = allFiles.Count(); this.Files = new TransferMetadata[this.FileCount]; // explicitly set the thread pool start amount to at most 500 int threadCount = Math.Min(this.FileCount, 500); var threads = new List <Thread>(threadCount); //start a bunch of new threads that will create the metadata and ensure a protected index. int currentIndex = 0; object indexIncrementLock = new object(); for (int i = 0; i < threadCount; i++) { var t = new Thread(() => { string curFile; while (allFiles.TryDequeue(out curFile)) { try { var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/'); var paramsPerFile = new TransferParameters ( curFile, String.Format("{0}{1}{2}", this.TargetStreamFolderPath, transferParameters.IsDownload ? "\\" : "/", relativeFilePath), transferParameters.AccountName, transferParameters.PerFileThreadCount, transferParameters.ConcurrentFileCount, transferParameters.IsOverwrite, transferParameters.IsResume, transferParameters.IsBinary, transferParameters.IsRecursive, transferParameters.IsDownload, transferParameters.MaxSegementLength, transferParameters.LocalMetadataLocation ); long size = -1; if (transferParameters.IsDownload && downloadFiles != null) { size = downloadFiles[curFile]; } var transferMetadataPath = Path.Combine(transferParameters.LocalMetadataLocation, string.Format("{0}.transfer.xml", Path.GetFileName(curFile))); var eachFileMetadata = new TransferMetadata(transferMetadataPath, paramsPerFile, frontend, size); lock (indexIncrementLock) { this.Files[currentIndex] = eachFileMetadata; currentIndex++; } } catch (Exception e) { exceptions.Enqueue(e); } } }); t.Start(); threads.Add(t); } foreach (var t in threads) { t.Join(); } if (exceptions.Count > 0) { throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray()); } }
/// <summary> /// Creates a new MultipleSegmentUploader. /// </summary> /// <param name="uploadMetadata">The metadata that keeps track of the file upload.</param> /// <param name="maxThreadCount">The maximum number of threads to use. Note that in some cases, this number may not be reached.</param> /// <param name="frontEnd">A pointer to the Front End interface to perform the upload to.</param> /// <param name="progressTracker">(Optional)A tracker that reports progress on each segment.</param> public MultipleSegmentUploader(UploadMetadata uploadMetadata, int maxThreadCount, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) : this(uploadMetadata, maxThreadCount, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, IProgress<UploadProgress> progressTracker = null) : this(uploadParameters, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new instance of the UploadMetadataGenerator with the given parameters and the given maximum append length. /// </summary> /// <param name="parameters">The parameters.</param> /// <param name="frontend">The frontend to use when generating new metadata.</param> public UploadMetadataGenerator(UploadParameters parameters, IFrontEndAdapter frontend) { _parameters = parameters; _frontend = frontend; }
/// <summary> /// Creates a new uploader for a single segment. /// </summary> /// <param name="segmentNumber">The sequence number of the segment.</param> /// <param name="uploadMetadata">The metadata for the entire upload.</param> /// <param name="frontEnd">A pointer to the front end.</param> /// <param name="progressTracker">(Optional) A tracker to report progress on this segment.</param> public SingleSegmentUploader(int segmentNumber, UploadMetadata uploadMetadata, IFrontEndAdapter frontEnd, IProgress<SegmentUploadProgress> progressTracker = null) : this(segmentNumber, uploadMetadata, frontEnd, CancellationToken.None, progressTracker) { }