// Upload a file
public static void UploadFile(string srcFilePath, string destFilePath, bool force = true)
{
var parameters = new UploadParameters(srcFilePath, destFilePath, _adlsAccountName, isOverwrite: force);
var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient);
var uploader = new DataLakeStoreUploader(parameters, frontend);
uploader.Execute();
}
public static bool DownloadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false)
{
var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, isDownload: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive);
var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient);
var uploader = new DataLakeStoreUploader(parameters, frontend);
uploader.Execute();
return(true);
}
// Upload the file
public static void UploadFile(string srcFilePath, string destFilePath, bool force = true)
{
//TODO: 5. Configure the file upload
var parameters = new UploadParameters(/*...*/);
var frontend = new DataLakeStoreFrontEndAdapter(/*...*/);
var uploader = new DataLakeStoreUploader(/*...*/);
//TODO: 6. Upload the file
uploader.//...
}
public void UploadFile(string srcFilePath, string destFilePath, bool force = true)
{
var ok = false;
while (!ok)
{
try {
_adlsFileSystemClient.FileSystem.Mkdirs(_adlsAccountName, destFilePath);
var parameters = new UploadParameters(srcFilePath, destFilePath, _adlsAccountName, isOverwrite: force);
var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient);
progress = new MyProgress();
var uploader = new DataLakeStoreUploader(parameters, frontend, progress);
uploader.Execute();
ok = true;
} catch (Exception ex) {
Console.WriteLine(ex.ToString());
Login();
}
}
}
private static void Main(string[] args)
{
try
{
Console.WriteLine("------------------------------------------------------------------------------");
Console.WriteLine($"AdlTransfer {Assembly.GetEntryAssembly().GetName().Version} Copyright (c) 2016 Sascha Dittmann. All Rights Reserved.");
Console.WriteLine("------------------------------------------------------------------------------");
if (ParseArguments(args))
{
return;
}
Console.WriteLine();
Console.WriteLine($"Source: {_sourcePath}");
Console.WriteLine($"Target: {_targetPath}");
Console.WriteLine($"Account Name: {_accountName}");
if (_verbose)
{
Console.WriteLine();
Console.WriteLine($"Per File Thread Count: {_perFileThreadCount}");
Console.WriteLine($"Concurrent File Count: {_concurrentFileCount}");
Console.WriteLine($"Segment Length: {_maxSegmentLength.ToSizeString()}");
Console.WriteLine();
Console.WriteLine($"Overwrite: {_isOverwrite}");
Console.WriteLine($"Binary: {_isBinary}");
Console.WriteLine($"Recursive: {_isRecursive}");
}
Console.WriteLine();
var credentials = Authenticate();
var client = new DataLakeStoreFileSystemManagementClient(credentials);
var frontEndAdapter = new DataLakeStoreFrontEndAdapter(_accountName, client);
var uploadParameters = new UploadParameters(
_sourcePath,
_targetPath,
_accountName,
_perFileThreadCount,
_concurrentFileCount,
_isOverwrite,
_isResume,
_isBinary,
_isRecursive,
_isDownload,
_maxSegmentLength,
_localMetadataLocation);
var progressTracker = new Progress <UploadProgress>();
progressTracker.ProgressChanged += UploadProgressChanged;
var folderProgressTracker = new Progress <UploadFolderProgress>();
folderProgressTracker.ProgressChanged += UploadFolderProgressChanged;
var uploader = new DataLakeStoreUploader(uploadParameters, frontEndAdapter, progressTracker, folderProgressTracker);
Console.WriteLine($"{(_isResume ? "Resuming" : "Starting")} {(_isDownload ? "Download" : "Upload")}...");
uploader.Execute();
Console.WriteLine($"{(_isDownload ? "Download" : "Upload")} completed.");
}
catch (Exception e)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.Error.WriteLine(e.Message);
Console.ResetColor();
Environment.ExitCode = -1;
}
}
public static bool UploadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false, bool testCancel = false)
{
var cancelSource = new CancellationTokenSource();
var myToken = cancelSource.Token;
var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive);
var progressTracker = new System.Progress <UploadFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
if (e.TotalFileCount == 0)
{
Console.WriteLine("we are done!");
}
};
var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient, myToken);
var uploader = new DataLakeStoreUploader(parameters, frontend, myToken, folderProgressTracker: progressTracker);
if (testCancel)
{
var uploadTask = Task.Run(() =>
{
myToken.ThrowIfCancellationRequested();
uploader.Execute();
myToken.ThrowIfCancellationRequested();
}, myToken);
try
{
while (!uploadTask.IsCompleted && !uploadTask.IsCanceled)
{
if (myToken.IsCancellationRequested)
{
// we are done tracking progress and will just break and let the task clean itself up.
try
{
uploadTask.Wait();
}
catch (OperationCanceledException)
{
if (uploadTask.IsCanceled)
{
uploadTask.Dispose();
}
}
catch (AggregateException ex)
{
if (ex.InnerExceptions.OfType <OperationCanceledException>().Any())
{
if (uploadTask.IsCanceled)
{
uploadTask.Dispose();
}
}
else
{
throw;
}
}
catch (Exception ex)
{
// swallow this for debugging to see what it is.
}
break;
}
Thread.Sleep(60000);
// run for 60 seconds and then cancel out and see what happens
cancelSource.Cancel();
}
}
catch (OperationCanceledException)
{
// do nothing since we successfully cancelled out
}
catch (Exception ex)
{
// see what the heck is going on.
}
}
else
{
uploader.Execute();
}
return(true);
}