/// <summary>
/// Verifies the progress status.
/// </summary>
/// <param name="progress">The progress.</param>
/// <param name="totalFileLength">Total length of the file.</param>
/// <param name="totalFiles">The total files.</param>
private void VerifyFolderProgressStatus(TransferFolderProgress progress, long totalFileLength, int totalFiles)
{
Assert.Equal(totalFileLength, progress.TotalFileLength);
Assert.Equal(totalFiles, progress.TotalFileCount);
Assert.Equal(progress.TotalFileCount, progress.TransferredFileCount);
// there are times where the operation will complete successfully before the final bytes update call back can run.
// We are not as concerned with the progress state being exactly correct, since validation of transferred bytes
// should really be checked against the actual file/folder being transfered and not on the progress callback.
// TODO: In the future, make the callback more robust and ensure that it always updates to include the final state.
Assert.InRange(progress.TransferredByteCount, progress.TotalFileLength * .8, progress.TotalFileLength);
for (int i = 0; i < progress.TotalFileCount; i++)
{
var eachProgress = progress.GetSegmentProgress(i);
VerifyProgressStatus(eachProgress, eachProgress.TotalFileLength);
}
}
public void DataLakeUploader_FreshFolderUploadDownload()
{
var frontEnd = new InMemoryFrontEnd();
var up = CreateParameters(isResume: false, isRecursive: true);
// set the per file thread count to the default to validate computed values.
up.PerFileThreadCount = -1;
TransferFolderProgress progress = null;
var syncRoot = new object();
IProgress <TransferFolderProgress> progressTracker = new Progress <TransferFolderProgress>(
(p) =>
{
lock (syncRoot)
{
//it is possible that these come out of order because of race conditions (multiple threads reporting at the same time); only update if we are actually making progress
if (progress == null || progress.TransferredByteCount < p.TransferredByteCount)
{
progress = p;
}
}
});
var uploader = new DataLakeStoreTransferClient(up, frontEnd, null, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, frontEnd);
VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
// verify that per file thread count is different but concurrent is the same.
Assert.True(up.PerFileThreadCount > 0 && uploader.Parameters.PerFileThreadCount > 0);
Assert.Equal(2, uploader.Parameters.ConcurrentFileCount);
// now download
var downloadFrontEnd = new MockableFrontEnd(frontEnd);
// replace the isDirectory implementation to return true
downloadFrontEnd.IsDirectoryImplementation = (streamPath) => { return(true); };
progress = null;
up = CreateParameters(isRecursive: true, isResume: false, isDownload: true, targetStreamPath: Path.GetDirectoryName(_downloadFilePath), isOverwrite: true, filePath: TargetStreamPath);
// set concurrentFileCount to default and validate that it changed.
up.ConcurrentFileCount = -1;
uploader = new DataLakeStoreTransferClient(up, downloadFrontEnd, null, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, downloadFrontEnd.BaseAdapter);
VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
Assert.True(up.ConcurrentFileCount > 0 && uploader.Parameters.ConcurrentFileCount > 0);
Assert.Equal(ThreadCount, uploader.Parameters.PerFileThreadCount);
// run it one more time with both as defaults
up.PerFileThreadCount = -1;
up.ConcurrentFileCount = -1;
uploader = new DataLakeStoreTransferClient(up, downloadFrontEnd, null, progressTracker);
uploader.Execute();
VerifyFileUploadedSuccessfully(up, downloadFrontEnd.BaseAdapter);
VerifyFolderProgressStatus(progress, _largeFileData.Length + (_smallFileData.Length * 2), 3);
Assert.True(up.ConcurrentFileCount > 0 && uploader.Parameters.ConcurrentFileCount > 0);
Assert.True(up.PerFileThreadCount > 0 && uploader.Parameters.PerFileThreadCount > 0);
}
