// Upload a file public static void UploadFile(string srcFilePath, string destFilePath, bool force = true) { var parameters = new UploadParameters(srcFilePath, destFilePath, _adlsAccountName, isOverwrite: force); var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient); var uploader = new DataLakeStoreUploader(parameters, frontend); uploader.Execute(); }
public static bool DownloadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false) { var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, isDownload: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive); var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient); var uploader = new DataLakeStoreUploader(parameters, frontend); uploader.Execute(); return(true); }
// Upload the file public static void UploadFile(string srcFilePath, string destFilePath, bool force = true) { //TODO: 5. Configure the file upload var parameters = new UploadParameters(/*...*/); var frontend = new DataLakeStoreFrontEndAdapter(/*...*/); var uploader = new DataLakeStoreUploader(/*...*/); //TODO: 6. Upload the file uploader.//... }
public void UploadFile(string srcFilePath, string destFilePath, bool force = true) { var ok = false; while (!ok) { try { _adlsFileSystemClient.FileSystem.Mkdirs(_adlsAccountName, destFilePath); var parameters = new UploadParameters(srcFilePath, destFilePath, _adlsAccountName, isOverwrite: force); var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient); progress = new MyProgress(); var uploader = new DataLakeStoreUploader(parameters, frontend, progress); uploader.Execute(); ok = true; } catch (Exception ex) { Console.WriteLine(ex.ToString()); Login(); } } }
private static void Main(string[] args) { try { Console.WriteLine("------------------------------------------------------------------------------"); Console.WriteLine($"AdlTransfer {Assembly.GetEntryAssembly().GetName().Version} Copyright (c) 2016 Sascha Dittmann. All Rights Reserved."); Console.WriteLine("------------------------------------------------------------------------------"); if (ParseArguments(args)) { return; } Console.WriteLine(); Console.WriteLine($"Source: {_sourcePath}"); Console.WriteLine($"Target: {_targetPath}"); Console.WriteLine($"Account Name: {_accountName}"); if (_verbose) { Console.WriteLine(); Console.WriteLine($"Per File Thread Count: {_perFileThreadCount}"); Console.WriteLine($"Concurrent File Count: {_concurrentFileCount}"); Console.WriteLine($"Segment Length: {_maxSegmentLength.ToSizeString()}"); Console.WriteLine(); Console.WriteLine($"Overwrite: {_isOverwrite}"); Console.WriteLine($"Binary: {_isBinary}"); Console.WriteLine($"Recursive: {_isRecursive}"); } Console.WriteLine(); var credentials = Authenticate(); var client = new DataLakeStoreFileSystemManagementClient(credentials); var frontEndAdapter = new DataLakeStoreFrontEndAdapter(_accountName, client); var uploadParameters = new UploadParameters( _sourcePath, _targetPath, _accountName, _perFileThreadCount, _concurrentFileCount, _isOverwrite, _isResume, _isBinary, _isRecursive, _isDownload, _maxSegmentLength, _localMetadataLocation); var progressTracker = new Progress <UploadProgress>(); progressTracker.ProgressChanged += UploadProgressChanged; var folderProgressTracker = new Progress <UploadFolderProgress>(); folderProgressTracker.ProgressChanged += UploadFolderProgressChanged; var uploader = new DataLakeStoreUploader(uploadParameters, frontEndAdapter, progressTracker, folderProgressTracker); Console.WriteLine($"{(_isResume ? "Resuming" : "Starting")} {(_isDownload ? "Download" : "Upload")}..."); uploader.Execute(); Console.WriteLine($"{(_isDownload ? "Download" : "Upload")} completed."); } catch (Exception e) { Console.ForegroundColor = ConsoleColor.Red; Console.Error.WriteLine(e.Message); Console.ResetColor(); Environment.ExitCode = -1; } }
public static bool UploadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false, bool testCancel = false) { var cancelSource = new CancellationTokenSource(); var myToken = cancelSource.Token; var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive); var progressTracker = new System.Progress <UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { if (e.TotalFileCount == 0) { Console.WriteLine("we are done!"); } }; var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient, myToken); var uploader = new DataLakeStoreUploader(parameters, frontend, myToken, folderProgressTracker: progressTracker); if (testCancel) { var uploadTask = Task.Run(() => { myToken.ThrowIfCancellationRequested(); uploader.Execute(); myToken.ThrowIfCancellationRequested(); }, myToken); try { while (!uploadTask.IsCompleted && !uploadTask.IsCanceled) { if (myToken.IsCancellationRequested) { // we are done tracking progress and will just break and let the task clean itself up. try { uploadTask.Wait(); } catch (OperationCanceledException) { if (uploadTask.IsCanceled) { uploadTask.Dispose(); } } catch (AggregateException ex) { if (ex.InnerExceptions.OfType <OperationCanceledException>().Any()) { if (uploadTask.IsCanceled) { uploadTask.Dispose(); } } else { throw; } } catch (Exception ex) { // swallow this for debugging to see what it is. } break; } Thread.Sleep(60000); // run for 60 seconds and then cancel out and see what happens cancelSource.Cancel(); } } catch (OperationCanceledException) { // do nothing since we successfully cancelled out } catch (Exception ex) { // see what the heck is going on. } } else { uploader.Execute(); } return(true); }