/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, CancellationToken token, IProgress <UploadProgress> progressTracker = null) { this.Parameters = uploadParameters; _frontEnd = frontEnd; //ensure that input parameters are correct ValidateParameters(); _metadataFilePath = GetCanonicalMetadataFilePath(); _progressTracker = progressTracker; _token = token; }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, CancellationToken token, IProgress<UploadProgress> progressTracker = null ) { this.Parameters = uploadParameters; _frontEnd = frontEnd; //ensure that input parameters are correct ValidateParameters(); _metadataFilePath = GetCanonicalMetadataFilePath(); _progressTracker = progressTracker; _token = token; }
/// <summary> /// Constructs a new UploadMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="uploadParameters">The parameters to use for constructing this metadata.</param> internal UploadMetadata(string metadataFilePath, UploadParameters uploadParameters) { this.MetadataFilePath = metadataFilePath; this.UploadId = Guid.NewGuid().ToString("N"); this.InputFilePath = uploadParameters.InputFilePath; this.TargetStreamPath = uploadParameters.TargetStreamPath; string streamDirectory; var streamName = SplitTargetStreamPathByName(out streamDirectory); if (string.IsNullOrEmpty(streamDirectory)) { // the scenario where the file is being uploaded at the root this.SegmentStreamDirectory = string.Format("/{0}.segments.{1}", streamName, Guid.NewGuid()); } else { // the scenario where the file is being uploaded in a sub folder this.SegmentStreamDirectory = string.Format("{0}/{1}.segments.{2}", streamDirectory, streamName, Guid.NewGuid()); } this.IsBinary = uploadParameters.IsBinary; var fileInfo = new FileInfo(uploadParameters.InputFilePath); this.FileLength = fileInfo.Length; // we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic. // this protects us against agressive increase of thread count resulting in far more segments than // is reasonable for a given file size. We also ensure that each segment is at least 256mb in size. // This is the size that ensures we have the optimal storage creation in the store. var preliminarySegmentCount = (int)Math.Ceiling((double)fileInfo.Length / uploadParameters.MaxSegementLength); this.SegmentCount = Math.Min(preliminarySegmentCount, UploadSegmentMetadata.CalculateSegmentCount(fileInfo.Length)); this.SegmentLength = UploadSegmentMetadata.CalculateSegmentLength(fileInfo.Length, this.SegmentCount); this.Segments = new UploadSegmentMetadata[this.SegmentCount]; for (int i = 0; i < this.SegmentCount; i++) { this.Segments[i] = new UploadSegmentMetadata(i, this); } }
/// <summary> /// Constructs a new UploadMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="uploadParameters">The parameters to use for constructing this metadata.</param> internal UploadMetadata(string metadataFilePath, UploadParameters uploadParameters) { this.MetadataFilePath = metadataFilePath; this.UploadId = Guid.NewGuid().ToString("N"); this.InputFilePath = uploadParameters.InputFilePath; this.TargetStreamPath = uploadParameters.TargetStreamPath; string streamDirectory; var streamName = SplitTargetStreamPathByName(out streamDirectory); if (string.IsNullOrEmpty(streamDirectory)) { // the scenario where the file is being uploaded at the root this.SegmentStreamDirectory = string.Format("/{0}.segments.{1}", streamName, Guid.NewGuid()); } else { // the scenario where the file is being uploaded in a sub folder this.SegmentStreamDirectory = string.Format("{0}/{1}.segments.{2}", streamDirectory, streamName, Guid.NewGuid()); } this.IsBinary = uploadParameters.IsBinary; var fileInfo = new FileInfo(uploadParameters.InputFilePath); this.FileLength = fileInfo.Length; // we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic. // this protects us against agressive increase of thread count resulting in far more segments than // is reasonable for a given file size. We also ensure that each segment is at least 256mb in size. // This is the size that ensures we have the optimal storage creation in the store. var preliminarySegmentCount = (int)Math.Ceiling((double) fileInfo.Length/uploadParameters.MaxSegementLength); this.SegmentCount = Math.Min(preliminarySegmentCount, UploadSegmentMetadata.CalculateSegmentCount(fileInfo.Length)); this.SegmentLength = UploadSegmentMetadata.CalculateSegmentLength(fileInfo.Length, this.SegmentCount); this.Segments = new UploadSegmentMetadata[this.SegmentCount]; for (int i = 0; i < this.SegmentCount; i++) { this.Segments[i] = new UploadSegmentMetadata(i, this); } }
/// <summary> /// Constructs a new UploadMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="uploadParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontEnd">The front end. This is used only in the constructor for determining file length</param> internal UploadMetadata(string metadataFilePath, UploadParameters uploadParameters, IFrontEndAdapter frontEnd, long fileSize = -1) { this.MetadataFilePath = metadataFilePath; this.UploadId = Guid.NewGuid().ToString("N"); this.InputFilePath = uploadParameters.InputFilePath; this.TargetStreamPath = uploadParameters.TargetStreamPath; this.IsDownload = uploadParameters.IsDownload; this.SegmentStreamDirectory = GetSegmentStreamDirectory(); this.IsBinary = uploadParameters.IsBinary; this.FileLength = fileSize < 0 ? frontEnd.GetStreamLength(uploadParameters.InputFilePath, !IsDownload) : fileSize; this.EncodingCodePage = uploadParameters.FileEncoding.CodePage; // we are taking the smaller number of segments between segment lengths of 256 and the segment growth logic. // this protects us against agressive increase of thread count resulting in far more segments than // is reasonable for a given file size. We also ensure that each segment is at least 256mb in size. // This is the size that ensures we have the optimal storage creation in the store. var preliminarySegmentCount = (int)Math.Ceiling((double)this.FileLength / uploadParameters.MaxSegementLength); this.SegmentCount = Math.Min(preliminarySegmentCount, UploadSegmentMetadata.CalculateSegmentCount(this.FileLength)); this.SegmentLength = UploadSegmentMetadata.CalculateSegmentLength(this.FileLength, this.SegmentCount); this.Segments = new UploadSegmentMetadata[this.SegmentCount]; for (int i = 0; i < this.SegmentCount; i++) { this.Segments[i] = new UploadSegmentMetadata(i, this); } if (!uploadParameters.IsBinary && this.SegmentCount > 1 && !this.IsDownload) { this.AlignSegmentsToRecordBoundaries(); // ensure that nothing strange happened during alignment this.ValidateConsistency(); } // initialize the status to pending, since it is not yet done. this.Status = SegmentUploadStatus.Pending; }
public void UploadMetadataGenerator_AlignSegmentsToRecordBoundaries() { //We keep creating a file, by appending a number of bytes to it (taken from FileLengthsInMB). //At each iteration, we append a new blob of data, and then run the whole test on the entire file var rnd = new Random(0); string folderPath = string.Format(@"{0}\uploadtest", Environment.CurrentDirectory); string filePath = Path.Combine(folderPath, "verifymetadata.txt"); try { if (!Directory.Exists(folderPath)) { Directory.CreateDirectory(folderPath); } if (File.Exists(filePath)) { File.Delete(filePath); } foreach (var lengthMB in FileLengthsMB) { var appendLength = (int) (lengthMB*1024*1024); AppendToFile(filePath, appendLength, rnd, 0, MaxAppendLength); string metadataFilePath = filePath + ".metadata.txt"; var up = new UploadParameters(filePath, filePath, null, isBinary: false, maxSegmentLength: 4*1024*1024); var mg = new UploadMetadataGenerator(up, MaxAppendLength); var metadata = mg.CreateNewMetadata(metadataFilePath); VerifySegmentsAreOnRecordBoundaries(metadata, filePath); } } finally { if (Directory.Exists(folderPath)) { Directory.Delete(folderPath, true); } } }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, IProgress<UploadProgress> progressTracker = null) : this(uploadParameters, frontEnd, CancellationToken.None, progressTracker) { }
/// <summary> /// Creates a new instance of the UploadMetadataGenerator with the given parameters and the given maximum append length. /// </summary> /// <param name="parameters">The parameters.</param> /// <param name="frontend">The frontend to use when generating new metadata.</param> public UploadMetadataGenerator(UploadParameters parameters, IFrontEndAdapter frontend) { _parameters = parameters; _frontend = frontend; }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int folderThreadCount = -1, int perFileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount; var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
/// <summary> /// Creates a new instance of the UploadMetadataGenerator with the given parameters and the default maximum append length. /// </summary> /// <param name="parameters">The parameters.</param> public UploadMetadataGenerator(UploadParameters parameters) : this(parameters, SingleSegmentUploader.BufferLength) { }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = 10, bool overwrite = false, bool resume = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { var previousTracing = ServiceClientTracing.IsEnabled; try { // disable this due to performance issues during download until issue: https://github.com/Azure/azure-powershell/issues/2499 is resolved. ServiceClientTracing.IsEnabled = false; FileType ignoredType; if (!overwrite && (!isDownload && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType) || (isDownload && File.Exists(destinationPath)))) { throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath)); } if (threadCount < 1) { threadCount = 10; // 10 is the default per our documentation. } // Progress bar indicator. var description = string.Format("Copying {0} File: {1} {2} Location: {3} for account: {4}", isDownload ? "Data Lake Store" : "Local", sourcePath, isDownload ? "to local" : "to Data Lake Store", destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("{0} Data Lake Store Store", isDownload ? "Download from" : "Upload to"), description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isDownload: isDownload); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.Expect100Continue = false; // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.Expect100Continue = previousExpect100; } } finally { ServiceClientTracing.IsEnabled = previousTracing; } }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int concurrentFileCount = 5, int perFileThreadCount = 10, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, isDownload, accountName); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, isDownload, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = concurrentFileCount <= 0 ? 5 : concurrentFileCount; var internalFileThreads = perFileThreadCount <= 0 ? 10 : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; var previousTracing = ServiceClientTracing.IsEnabled; try { // disable this due to performance issues during download until issue: https://github.com/Azure/azure-powershell/issues/2499 is resolved. ServiceClientTracing.IsEnabled = false; ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive, isDownload: isDownload); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServiceClientTracing.IsEnabled = previousTracing; ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { var originalValue = TracingAdapter.IsEnabled; try { //TODO: Remove this logic when defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) is resolved TracingAdapter.IsEnabled = false; // default the number of threads to use to the processor count if (threadCount < 1) { threadCount = Environment.ProcessorCount; } // Progress bar indicator. var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}", sourcePath, destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), "Upload to DataLakeStore Store", description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100); } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, overwrite, resume, isBinary); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } finally { TracingAdapter.IsEnabled = originalValue; } }
/// <summary> /// Constructs a new UploadMetadata from the given parameters. /// </summary> /// <param name="metadataFilePath">The file path to assign to this metadata file (for saving purposes).</param> /// <param name="uploadParameters">The parameters to use for constructing this metadata.</param> /// <param name="frontend">The frontend to use when generating per file metadata.</param> public UploadFolderMetadata(string metadataFilePath, UploadParameters uploadParameters, IFrontEndAdapter frontend) { this.MetadataFilePath = metadataFilePath; this.UploadId = Guid.NewGuid().ToString("N"); this.InputFolderPath = uploadParameters.InputFilePath; this.TargetStreamFolderPath = uploadParameters.TargetStreamPath.TrimEnd('/'); this.IsRecursive = uploadParameters.IsRecursive; // get this list of all files in the source directory, depending on if this is recursive or not. ConcurrentQueue <string> allFiles; ConcurrentQueue <Exception> exceptions = new ConcurrentQueue <Exception>(); Dictionary <string, long> downloadFiles = new Dictionary <string, long>(); if (uploadParameters.IsDownload) { foreach (var entry in frontend.ListDirectory(uploadParameters.InputFilePath, uploadParameters.IsRecursive)) { downloadFiles.Add(entry.Key, entry.Value); } allFiles = new ConcurrentQueue <string>(downloadFiles.Keys); this.TotalFileBytes = downloadFiles.Values.Sum(); } else { allFiles = new ConcurrentQueue <string>(this.IsRecursive ? Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.AllDirectories) : Directory.EnumerateFiles(this.InputFolderPath, "*.*", SearchOption.TopDirectoryOnly)); this.TotalFileBytes = GetByteCountFromFileList(allFiles); } this.FileCount = allFiles.Count(); this.Files = new UploadMetadata[this.FileCount]; // explicitly set the thread pool start amount to at most 500 int threadCount = Math.Min(this.FileCount, 500); var threads = new List <Thread>(threadCount); //start a bunch of new threads that will create the metadata and ensure a protected index. int currentIndex = 0; object indexIncrementLock = new object(); for (int i = 0; i < threadCount; i++) { var t = new Thread(() => { string curFile; while (allFiles.TryDequeue(out curFile)) { try { var relativeFilePath = curFile.Replace(this.InputFolderPath, "").TrimStart('\\').TrimStart('/'); var paramsPerFile = new UploadParameters ( curFile, String.Format("{0}{1}{2}", this.TargetStreamFolderPath, uploadParameters.IsDownload ? "\\" : "/", relativeFilePath), uploadParameters.AccountName, uploadParameters.PerFileThreadCount, uploadParameters.ConcurrentFileCount, uploadParameters.IsOverwrite, uploadParameters.IsResume, uploadParameters.IsBinary, uploadParameters.IsRecursive, uploadParameters.IsDownload, uploadParameters.MaxSegementLength, uploadParameters.LocalMetadataLocation ); long size = -1; if (uploadParameters.IsDownload && downloadFiles != null) { size = downloadFiles[curFile]; } var uploadMetadataPath = Path.Combine(uploadParameters.LocalMetadataLocation, string.Format("{0}.upload.xml", Path.GetFileName(curFile))); var eachFileMetadata = new UploadMetadata(uploadMetadataPath, paramsPerFile, frontend, size); lock (indexIncrementLock) { this.Files[currentIndex] = eachFileMetadata; currentIndex++; } } catch (Exception e) { exceptions.Enqueue(e); } } }); t.Start(); threads.Add(t); } foreach (var t in threads) { t.Join(); } if (exceptions.Count > 0) { throw new AggregateException("At least one file failed to have metadata generated", exceptions.ToArray()); } }
public static bool UploadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false, bool testCancel = false) { var cancelSource = new CancellationTokenSource(); var myToken = cancelSource.Token; var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive); var progressTracker = new System.Progress <UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { if (e.TotalFileCount == 0) { Console.WriteLine("we are done!"); } }; var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient, myToken); var uploader = new DataLakeStoreUploader(parameters, frontend, myToken, folderProgressTracker: progressTracker); if (testCancel) { var uploadTask = Task.Run(() => { myToken.ThrowIfCancellationRequested(); uploader.Execute(); myToken.ThrowIfCancellationRequested(); }, myToken); try { while (!uploadTask.IsCompleted && !uploadTask.IsCanceled) { if (myToken.IsCancellationRequested) { // we are done tracking progress and will just break and let the task clean itself up. try { uploadTask.Wait(); } catch (OperationCanceledException) { if (uploadTask.IsCanceled) { uploadTask.Dispose(); } } catch (AggregateException ex) { if (ex.InnerExceptions.OfType <OperationCanceledException>().Any()) { if (uploadTask.IsCanceled) { uploadTask.Dispose(); } } else { throw; } } catch (Exception ex) { // swallow this for debugging to see what it is. } break; } Thread.Sleep(60000); // run for 60 seconds and then cancel out and see what happens cancelSource.Cancel(); } } catch (OperationCanceledException) { // do nothing since we successfully cancelled out } catch (Exception ex) { // see what the heck is going on. } } else { uploader.Execute(); } return(true); }
/// <summary> /// Creates a new instance of the UploadMetadataGenerator with the given parameters and the default maximum append length. /// </summary> /// <param name="parameters">The parameters.</param> public UploadMetadataGenerator(UploadParameters parameters) : this(parameters, SingleSegmentUploader.BufferLength) { }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { FileType ignoredType; if (!overwrite && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType)) { throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath)); } //TODO: defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) needs to be resolved or the tracingadapter work around needs to be put back in // default the number of threads to use to the processor count if (threadCount < 1) { threadCount = Environment.ProcessorCount; } // Progress bar indicator. var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}", sourcePath, destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), "Upload to DataLakeStore Store", description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100); } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, overwrite, resume, isBinary); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.Expect100Continue = false; // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } finally { ServicePointManager.Expect100Continue = previousExpect100; } }
/// <summary> /// Creates a new instance of the UploadMetadataGenerator with the given parameters and the given maximum append length. /// </summary> /// <param name="parameters"></param> /// <param name="maxAppendLength"></param> public UploadMetadataGenerator(UploadParameters parameters, int maxAppendLength) { _parameters = parameters; _maxAppendLength = maxAppendLength; }
/// <summary> /// Creates a new instance of the UploadMetadataGenerator with the given parameters and the given maximum append length. /// </summary> /// <param name="parameters"></param> /// <param name="maxAppendLength"></param> public UploadMetadataGenerator(UploadParameters parameters, int maxAppendLength) { _parameters = parameters; _maxAppendLength = maxAppendLength; }
/// <summary> /// Creates a new instance of the DataLakeUploader class, by specifying a pointer to the FrontEnd to use for the upload. /// </summary> /// <param name="uploadParameters">The Upload Parameters to use.</param> /// <param name="frontEnd">A pointer to the FrontEnd interface to use for the upload.</param> /// <param name="progressTracker">(Optional) A tracker that reports progress on the upload.</param> public DataLakeStoreUploader(UploadParameters uploadParameters, IFrontEndAdapter frontEnd, IProgress <UploadProgress> progressTracker = null) : this(uploadParameters, frontEnd, CancellationToken.None, progressTracker) { }