public void DataLakeUploader_UploadDownloadSingleSegment()
{
var frontEnd = new InMemoryFrontEnd();
var up = new TransferParameters(
inputFilePath: _smallFilePath,
targetStreamPath: "1",
perFileThreadCount: ThreadCount,
accountName: "foo",
isResume: false,
maxSegmentLength: 4 * 1024 * 1024,
localMetadataLocation: Path.GetTempPath());
File.WriteAllBytes(_smallFilePath, _smallFileData);
var uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
up = new TransferParameters(
inputFilePath: "1",
targetStreamPath: _downloadFilePath,
perFileThreadCount: ThreadCount,
accountName: "foo",
isResume: false,
isOverwrite: true,
isDownload: true,
maxSegmentLength: 4 * 1024 * 1024,
localMetadataLocation: Path.GetTempPath());
// now download
uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData);
}
public void DataLakeUploader_CancelUpload()
{
CancellationTokenSource myTokenSource = new CancellationTokenSource();
var cancelToken = myTokenSource.Token;
var frontEnd = new InMemoryFrontEnd();
var mockedFrontend = new MockableFrontEnd(frontEnd);
mockedFrontend.GetStreamLengthImplementation = (streamPath, isDownload) =>
{
// sleep for 2 second to allow for the cancellation to actual happen
Thread.Sleep(2000);
return(frontEnd.GetStreamLength(streamPath, isDownload));
};
mockedFrontend.StreamExistsImplementation = (streamPath, isDownload) =>
{
// sleep for 2 second to allow for the cancellation to actual happen
Thread.Sleep(2000);
return(frontEnd.StreamExists(streamPath, isDownload));
};
var up = CreateParameters(isResume: false);
TransferProgress progress = null;
var syncRoot = new object();
IProgress <TransferProgress> progressTracker = new Progress <TransferProgress>(
(p) =>
{
lock (syncRoot)
{
//it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
if (progress == null || progress.TransferredByteCount < p.TransferredByteCount)
{
progress = p;
}
}
});
var uploader = new DataLakeStoreTransferClient(up, mockedFrontend, cancelToken, progressTracker);
Task uploadTask = Task.Run(() =>
{
uploader.Execute();
Thread.Sleep(2000);
}, cancelToken);
myTokenSource.Cancel();
Assert.True(cancelToken.IsCancellationRequested);
while (uploadTask.Status == TaskStatus.Running || uploadTask.Status == TaskStatus.WaitingToRun)
{
Thread.Sleep(250);
}
// Verify that the file did not get uploaded completely.
Assert.False(frontEnd.StreamExists(up.TargetStreamPath), "Uploaded stream exists when it should not yet have been completely created");
}
public void DataLakeUploader_TargetExistsNoOverwrite()
{
var frontEnd = new InMemoryFrontEnd();
frontEnd.CreateStream(TargetStreamPath, true, null, 0);
//no resume, no overwrite
var up = CreateParameters(filePath: _smallFilePath, isResume: false);
var uploader = new DataLakeStoreTransferClient(up, frontEnd);
Assert.Throws <InvalidOperationException>(() => uploader.Execute());
//resume, no overwrite
up = CreateParameters(filePath: _smallFilePath, isResume: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
Assert.Throws <InvalidOperationException>(() => uploader.Execute());
//resume, overwrite
up = CreateParameters(filePath: _smallFilePath, isResume: true, isOverwrite: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
//no resume, overwrite
up = CreateParameters(filePath: _smallFilePath, isResume: false, isOverwrite: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
//download no resume, no overwrite
up = CreateParameters(filePath: TargetStreamPath, targetStreamPath: _downloadFilePath, isResume: false, isDownload: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
Assert.Throws <InvalidOperationException>(() => uploader.Execute());
//download resume, no overwrite
up = CreateParameters(filePath: TargetStreamPath, targetStreamPath: _downloadFilePath, isResume: true, isDownload: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
Assert.Throws <InvalidOperationException>(() => uploader.Execute());
//download resume, overwrite
up = CreateParameters(filePath: TargetStreamPath, targetStreamPath: _downloadFilePath, isResume: true, isOverwrite: true, isDownload: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
//download no resume, overwrite
up = CreateParameters(filePath: TargetStreamPath, targetStreamPath: _downloadFilePath, isResume: false, isOverwrite: true, isDownload: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.Execute();
}
public void DataLakeUploader_ResumePartialFolderUpload()
{
//attempt to load the file fully, but only allow creating 1 target stream
var backingFrontEnd = new InMemoryFrontEnd();
var frontEnd = new MockableFrontEnd(backingFrontEnd);
int createStreamCount = 0;
frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) =>
{
createStreamCount++;
if (createStreamCount > 1)
{
//we only allow 1 file to be created
throw new IntentionalException();
}
backingFrontEnd.CreateStream(path, overwrite, data, byteCount);
};
var up = CreateParameters(isResume: false, isRecursive: true);
var uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.DeleteMetadataFile();
Assert.Throws <AggregateException>(() => uploader.Execute());
Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count);
Assert.Equal(1, backingFrontEnd.StreamCount);
//resume the upload but point it to the real back-end, which doesn't throw exceptions
up = CreateParameters(isResume: true, isRecursive: true);
uploader = new DataLakeStoreTransferClient(up, backingFrontEnd);
try
{
uploader.Execute();
}
finally
{
uploader.DeleteMetadataFile();
}
VerifyFileUploadedSuccessfully(up, backingFrontEnd);
}
public void DataLakeUploader_FreshUploadDownload()
{
var frontEnd = new InMemoryFrontEnd();
var up = CreateParameters(isResume: false);
TransferProgress progress = null;
var syncRoot = new object();
IProgress <TransferProgress> progressTracker = new Progress <TransferProgress>(
(p) =>
{
lock (syncRoot)
{
//it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
if (progress == null || progress.TransferredByteCount < p.TransferredByteCount)
{
progress = p;
}
}
});
var uploader = new DataLakeStoreTransferClient(up, frontEnd, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd);
VerifyProgressStatus(progress, _largeFileData.Length);
Assert.Equal(ThreadCount, uploader.Parameters.PerFileThreadCount);
// now download
// change the thread count to be the default and validate that it is different.
progress = null;
up = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: TargetStreamPath);
up.PerFileThreadCount = -1;
uploader = new DataLakeStoreTransferClient(up, frontEnd, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd);
VerifyProgressStatus(progress, _largeFileData.Length);
Assert.True(up.PerFileThreadCount > 0 && uploader.Parameters.PerFileThreadCount > 0);
}
public void DataLakeUploader_ResumePartialUploadDownload()
{
//attempt to load the file fully, but only allow creating 1 target stream
var backingFrontEnd = new InMemoryFrontEnd();
var frontEnd = new MockableFrontEnd(backingFrontEnd);
int createStreamCount = 0;
frontEnd.CreateStreamImplementation = (path, overwrite, data, byteCount) =>
{
createStreamCount++;
if (createStreamCount > 1)
{
//we only allow 1 file to be created
throw new IntentionalException();
}
backingFrontEnd.CreateStream(path, overwrite, data, byteCount);
};
var up = CreateParameters(isResume: false);
var uploader = new DataLakeStoreTransferClient(up, frontEnd);
uploader.DeleteMetadataFile();
Assert.Throws <AggregateException>(() => uploader.Execute());
Assert.Equal(1, frontEnd.ListDirectory(up.TargetStreamPath, false).Keys.Count);
Assert.Equal(1, backingFrontEnd.StreamCount);
//resume the upload but point it to the real back-end, which doesn't throw exceptions
up = CreateParameters(isResume: true);
uploader = new DataLakeStoreTransferClient(up, backingFrontEnd);
try
{
uploader.Execute();
}
finally
{
uploader.DeleteMetadataFile();
}
VerifyFileUploadedSuccessfully(up, backingFrontEnd);
// now download the same way.
var frontEnd2 = new MockableFrontEnd(backingFrontEnd); // need to have data from the successful upload available.
createStreamCount = 0;
frontEnd2.ReadStreamImplementation = (path, data, byteCount, isDownload) =>
{
createStreamCount++;
if (createStreamCount > 1)
{
//we only allow 1 file to be created
throw new IntentionalException();
}
return(backingFrontEnd.ReadStream(path, data, byteCount, isDownload));
};
up = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.TargetStreamPath);
uploader = new DataLakeStoreTransferClient(up, frontEnd2);
Assert.Throws <AggregateException>(() => uploader.Execute());
Assert.False(frontEnd2.StreamExists(up.TargetStreamPath), "Target stream should not have been created");
// now use the good front end
up = CreateParameters(isResume: true, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.InputFilePath);
uploader = new DataLakeStoreTransferClient(up, backingFrontEnd);
//resume the download but point it to the real back-end, which doesn't throw exceptions
try
{
uploader.Execute();
}
finally
{
uploader.DeleteMetadataFile();
}
VerifyFileUploadedSuccessfully(up, backingFrontEnd);
}
public void DataLakeUploader_ResumeUploadDownloadWithAllMissingFiles()
{
//this scenario is achieved by refusing to execute the concat command on the front end for the initial upload (which will interrupt it)
//and then resuming the upload against a fresh front-end (which obviously has no files there)
var backingFrontEnd1 = new InMemoryFrontEnd();
var frontEnd1 = new MockableFrontEnd(backingFrontEnd1);
frontEnd1.ConcatenateImplementation = (target, inputs, isDownload) => { throw new IntentionalException(); }; //fail the concatenation
//attempt full upload
var up = CreateParameters(isResume: false);
var uploader = new DataLakeStoreTransferClient(up, frontEnd1);
uploader.DeleteMetadataFile();
Assert.Throws <IntentionalException>(() => uploader.Execute());
Assert.False(frontEnd1.StreamExists(up.TargetStreamPath), "Target stream should not have been created");
Assert.True(0 < backingFrontEnd1.StreamCount, "No temporary streams seem to have been created");
//attempt to resume the upload
var frontEnd2 = new InMemoryFrontEnd();
up = CreateParameters(isResume: true);
uploader = new DataLakeStoreTransferClient(up, frontEnd2);
//at this point the metadata exists locally but there are no target files in frontEnd2
try
{
uploader.Execute();
}
finally
{
uploader.DeleteMetadataFile();
}
VerifyFileUploadedSuccessfully(up, frontEnd2);
// now download the same way.
var frontEnd3 = new MockableFrontEnd(frontEnd2); // need to have data from the successful upload available.
frontEnd3.ConcatenateImplementation = (target, inputs, isDownload) => { throw new IntentionalException(); }; //fail the concatenation
up = CreateParameters(isResume: false, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.TargetStreamPath);
uploader = new DataLakeStoreTransferClient(up, frontEnd3);
Assert.Throws <IntentionalException>(() => uploader.Execute());
Assert.False(frontEnd1.StreamExists(up.TargetStreamPath, true), "Target stream should not have been created");
// now use the good front end
up = CreateParameters(isResume: true, isDownload: true, targetStreamPath: _downloadFilePath, isOverwrite: true, filePath: up.InputFilePath);
uploader = new DataLakeStoreTransferClient(up, frontEnd2);
//at this point the metadata exists locally but there are no target files in frontEnd2
try
{
uploader.Execute();
}
finally
{
uploader.DeleteMetadataFile();
}
VerifyFileUploadedSuccessfully(up, frontEnd2);
}
public void DataLakeUploader_FreshFolderUploadDownload()
{
var frontEnd = new InMemoryFrontEnd();
var up = CreateParameters(isResume: false, isRecursive: true);
// set the per file thread count to the default to validate computed values.
up.PerFileThreadCount = -1;
TransferFolderProgress progress = null;
var syncRoot = new object();
IProgress <TransferFolderProgress> progressTracker = new Progress <TransferFolderProgress>(
(p) =>
{
lock (syncRoot)
{
//it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
if (progress == null || progress.TransferredByteCount < p.TransferredByteCount)
{
progress = p;
}
}
});
var uploader = new DataLakeStoreTransferClient(up, frontEnd, null, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd);
VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
// verify that per file thread count is different but concurrent is the same.
Assert.True(up.PerFileThreadCount > 0 && uploader.Parameters.PerFileThreadCount > 0);
Assert.Equal(2, uploader.Parameters.ConcurrentFileCount);
// now download
var downloadFrontEnd = new MockableFrontEnd(frontEnd);
// replace the isDirectory implementation to return true
downloadFrontEnd.IsDirectoryImplementation = (streamPath) => { return(true); };
progress = null;
up = CreateParameters(isRecursive: true, isResume: false, isDownload: true, targetStreamPath: Path.GetDirectoryName(_downloadFilePath), isOverwrite: true, filePath: TargetStreamPath);
// set concurrentFileCount to default and validate that it changed.
up.ConcurrentFileCount = -1;
uploader = new DataLakeStoreTransferClient(up, downloadFrontEnd, null, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, downloadFrontEnd.BaseAdapter);
VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
Assert.True(up.ConcurrentFileCount > 0 && uploader.Parameters.ConcurrentFileCount > 0);
Assert.Equal(ThreadCount, uploader.Parameters.PerFileThreadCount);
// run it one more time with both as defaults
up.PerFileThreadCount = -1;
up.ConcurrentFileCount = -1;
uploader = new DataLakeStoreTransferClient(up, downloadFrontEnd, null, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, downloadFrontEnd.BaseAdapter);
VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
Assert.True(up.ConcurrentFileCount > 0 && uploader.Parameters.ConcurrentFileCount > 0);
Assert.True(up.PerFileThreadCount > 0 && uploader.Parameters.PerFileThreadCount > 0);
}
/// <summary>
/// Uploads a folder to the specified Data Lake Store account.
/// </summary>
/// <param name='accountName'>
/// The Azure Data Lake Store account to execute filesystem operations on.
/// </param>
/// <param name='sourcePath'>
/// The local source folder to upload to the Data Lake Store account.
/// </param>
/// <param name='destinationPath'>
/// The Data Lake Store path (starting with '/') of the directory to upload to.
/// </param>
/// <param name='perFileThreadCount'>
/// The maximum number of threads to use per file during the upload. By default, this number will be computed based on folder structure and average file size.
/// </param>
/// <param name='concurrentFileCount'>
/// The maximum number of files to upload at once. By default, this number will be computed based on folder structure and number of files.
/// </param>
/// <param name='resume'>
/// A switch indicating if this upload is a continuation of a previous, failed upload. Default is false.
/// </param>
/// <param name='overwrite'>
/// A switch indicating this upload should overwrite the contents of the target directory if it exists. Default is false, and the upload will fast fail if the target location exists.
/// </param>
/// <param name='uploadAsBinary'>
/// A switch indicating this upload should treat all data as binary, which is slightly more performant but does not ensure record boundary integrity. This is recommended for large folders of mixed binary and text files or binary only directories. Default is false
/// </param>
/// <param name='recurse'>
/// A switch indicating this upload should upload the source directory recursively or just the top level. Default is false, only the top level will be uploaded.
/// </param>
/// <param name='progressTracker'>
/// An optional delegate that can be used to track the progress of the upload operation asynchronously.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
/// <exception cref="AdlsErrorException">
/// Thrown when the operation returned an invalid status code.
/// </exception>
/// <exception cref="TaskCanceledException">
/// Thrown when the operation takes too long to complete or if the user explicitly cancels it.
/// </exception>
/// <exception cref="InvalidMetadataException">
/// Thrown when resume metadata is corrupt or not associated with the current operation.
/// </exception>
/// <exception cref="FileNotFoundException">
/// Thrown when the source path cannot be found.
/// </exception>
/// <exception cref="InvalidOperationException">
/// Thrown if an invalid upload is attempted or a file/folder is modified externally during the operation.
/// </exception>
/// <exception cref="TransferFailedException">
/// Thrown if the transfer operation fails.
/// </exception>
/// <exception cref="SerializationException">
/// Thrown when unable to deserialize the response
/// </exception>
/// <exception cref="ValidationException">
/// Thrown when a required parameter is null
/// </exception>
public void UploadFolder(
string accountName,
string sourcePath,
string destinationPath,
int perFileThreadCount = -1,
int concurrentFileCount = -1,
bool resume = false,
bool overwrite = false,
bool uploadAsBinary = false,
bool recurse = false,
IProgress <TransferFolderProgress> progressTracker = null,
CancellationToken cancellationToken = default(CancellationToken))
{
bool _shouldTrace = ServiceClientTracing.IsEnabled;
string _invocationId = null;
if (_shouldTrace)
{
_invocationId = ServiceClientTracing.NextInvocationId.ToString();
Dictionary <string, object> tracingParameters = new Dictionary <string, object>();
tracingParameters.Add("accountName", accountName);
tracingParameters.Add("sourcePath", sourcePath);
tracingParameters.Add("destinationPath", destinationPath);
tracingParameters.Add("perFileThreadCount", perFileThreadCount);
tracingParameters.Add("concurrentFileCount", concurrentFileCount);
tracingParameters.Add("resume", resume);
tracingParameters.Add("overwrite", overwrite);
tracingParameters.Add("recurse", recurse);
tracingParameters.Add("uploadAsBinary", uploadAsBinary);
tracingParameters.Add("progressTracker", progressTracker);
tracingParameters.Add("cancellationToken", cancellationToken);
ServiceClientTracing.Enter(_invocationId, this, "UploadFolder", tracingParameters);
}
try
{
var parameters = new TransferParameters(
inputFilePath: sourcePath,
targetStreamPath: destinationPath,
accountName: accountName,
perFileThreadCount: perFileThreadCount,
concurrentFileCount: concurrentFileCount,
isOverwrite: overwrite,
isResume: resume,
isRecursive: recurse,
isBinary: uploadAsBinary
);
var transferAdapter = new DataLakeStoreFrontEndAdapter(accountName, this.Client);
var transferClient = new DataLakeStoreTransferClient(
parameters,
transferAdapter,
token: cancellationToken,
folderProgressTracker: progressTracker);
transferClient.Execute();
if (_shouldTrace)
{
ServiceClientTracing.Exit(
_invocationId,
string.Format(
"Upload of folder to account: {0} from source location: {1}{2} to destination: {3} completed successfully.",
accountName,
sourcePath,
recurse ? ", recursively," : string.Empty,
destinationPath));
}
}
catch (Exception ex)
{
if (_shouldTrace)
{
ServiceClientTracing.Error(_invocationId, ex);
}
throw ex;
}
}
/// <summary>
/// Downloads a file from the specified Data Lake Store account.
/// </summary>
/// <param name='accountName'>
/// The Azure Data Lake Store account to execute filesystem operations on.
/// </param>
/// <param name='sourcePath'>
/// The Data Lake Store path (starting with '/') of the file to download.
/// </param>
/// <param name='destinationPath'>
/// The local path to download the file to. If a directory is specified, the file name will be the same as the source file name
/// </param>
/// <param name='threadCount'>
/// The maximum number of threads to use during the download. By default, this number will be computed based on file size.
/// </param>
/// <param name='resume'>
/// A switch indicating if this download is a continuation of a previous, failed download. Default is false.
/// </param>
/// <param name='overwrite'>
/// A switch indicating this download should overwrite the the target file if it exists. Default is false, and the download will fast fail if the target file exists.
/// </param>
/// <param name='progressTracker'>
/// An optional delegate that can be used to track the progress of the download operation asynchronously.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
/// <exception cref="AdlsErrorException">
/// Thrown when the operation returned an invalid status code.
/// </exception>
/// <exception cref="TaskCanceledException">
/// Thrown when the operation takes too long to complete or if the user explicitly cancels it.
/// </exception>
/// <exception cref="InvalidMetadataException">
/// Thrown when resume metadata is corrupt or not associated with the current operation.
/// </exception>
/// <exception cref="FileNotFoundException">
/// Thrown when the source path cannot be found.
/// </exception>
/// <exception cref="InvalidOperationException">
/// Thrown if an invalid download is attempted or a file is modified externally during the operation.
/// </exception>
/// <exception cref="TransferFailedException">
/// Thrown if the transfer operation fails.
/// </exception>
/// <exception cref="SerializationException">
/// Thrown when unable to deserialize the response
/// </exception>
/// <exception cref="ValidationException">
/// Thrown when a required parameter is null
/// </exception>
public void DownloadFile(
string accountName,
string sourcePath,
string destinationPath,
int threadCount = -1,
bool resume = false,
bool overwrite = false,
IProgress <TransferProgress> progressTracker = null,
CancellationToken cancellationToken = default(CancellationToken))
{
bool _shouldTrace = ServiceClientTracing.IsEnabled;
string _invocationId = null;
if (_shouldTrace)
{
_invocationId = ServiceClientTracing.NextInvocationId.ToString();
Dictionary <string, object> tracingParameters = new Dictionary <string, object>();
tracingParameters.Add("accountName", accountName);
tracingParameters.Add("sourcePath", sourcePath);
tracingParameters.Add("destinationPath", destinationPath);
tracingParameters.Add("threadCount", threadCount);
tracingParameters.Add("resume", resume);
tracingParameters.Add("overwrite", overwrite);
tracingParameters.Add("progressTracker", progressTracker);
tracingParameters.Add("cancellationToken", cancellationToken);
ServiceClientTracing.Enter(_invocationId, this, "DownloadFile", tracingParameters);
}
try
{
var parameters = new TransferParameters(
inputFilePath: sourcePath,
targetStreamPath: destinationPath,
accountName: accountName,
perFileThreadCount: threadCount,
isOverwrite: overwrite,
isResume: resume,
isDownload: true
);
var transferAdapter = new DataLakeStoreFrontEndAdapter(accountName, this.Client);
var transferClient = new DataLakeStoreTransferClient(
parameters,
transferAdapter,
cancellationToken,
progressTracker);
transferClient.Execute();
if (_shouldTrace)
{
ServiceClientTracing.Exit(
_invocationId,
string.Format(
"Download of stream in account: {0} from source location: {1} to destination: {2} completed successfully.",
accountName,
sourcePath,
destinationPath));
}
}
catch (Exception ex)
{
if (_shouldTrace)
{
ServiceClientTracing.Error(_invocationId, ex);
}
throw ex;
}
}
