/// <summary>
/// Creates an entry to the internal dictionary for the new file. The entry encapsulates AclStatus, DirectoryEntry and a memory stream
/// </summary>
/// <param name="filename">File name</param>
/// <param name="mode">If exists hether to overwrite or fail</param>
/// <param name="octalPermission">Permission string</param>
/// <param name="createParent">True if we create parent directories- currently has no effect</param>
/// <returns>Mock ADls output stream</returns>
public override AdlsOutputStream CreateFile(string filename, IfExists mode, string octalPermission = null,
bool createParent = true)
{
if (mode == IfExists.Fail && _directoryEntries.ContainsKey(filename))
{
throw new AdlsException("The file exists");
}
var metaData = CreateMetaData(filename, DirectoryEntryType.FILE, octalPermission);
_directoryEntries.Add(filename, metaData);
return(new MockAdlsOutputStream(metaData.DataStream));
}
private FileUploader(string srcPath, string destPath, AdlsClient client, int numThreads,
IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool resume, bool isBinary, CancellationToken cancelToken, bool ingressTest, long chunkSize) : base(srcPath, destPath, client, numThreads, doOverwrite, progressTracker, notRecurse, resume, ingressTest, chunkSize, Path.Combine(Path.GetTempPath(), ".adl", "Upload", GetTransferLogFileName(srcPath, destPath, Path.DirectorySeparatorChar, '/')), cancelToken, $"binary:{isBinary}")
{
// If not recurse then we will have one thread and ProducerFirstPass logic loop will run only once
NumProducerThreads = NotRecurse ? 1 : NumProducerThreadsFirstPass;
UploaderProducerQueue = new QueueWrapper <DirectoryInfo>(NumProducerThreads);
if (FileTransferLog.IsDebugEnabled)
{
FileTransferLog.Debug($"FileTransfer.Uploader, Src: {SourcePath}, Dest: {DestPath}, Threads: {NumConsumerThreads}, TrackingProgress: {ProgressTracker != null}, OverwriteIfExist: {DoOverwrite == IfExists.Overwrite}");
}
_isBinary = isBinary;
_encodeType = Encoding.UTF8;
}
private FileDownloader(string srcPath, string destPath, AdlsClient client, int numThreads,
IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool disableTransferLogging, bool resume, CancellationToken cancelToken, bool egressTest, int egressBufferCapacity, long chunkSize) : base(srcPath, destPath, client, numThreads, doOverwrite, progressTracker, notRecurse, disableTransferLogging, resume, egressTest, chunkSize, Path.Combine(Path.GetTempPath(), ".adl", "Download", GetTransferLogFileName(client.AccountFQDN, srcPath, destPath, '/', Path.DirectorySeparatorChar)), cancelToken)
{
EgressBufferCapacity = egressBufferCapacity;
// If not recurse then we will have one thread and ProducerFirstPass logic loop will run only once
NumProducerThreads = notRecurse ? 1 : NumProducerThreadsFirstPass;
DownloaderProducerQueue = new QueueWrapper <DirectoryEntry>(NumProducerThreads);
DownloaderList = new List <DirectoryEntry>(DownloaderListCapacity);
if (FileTransferLog.IsDebugEnabled)
{
FileTransferLog.Debug($"FileTransfer.Downloader, Src: {SourcePath}, Dest: {DestPath}, Threads: {NumConsumerThreads}, TrackingProgress: {ProgressTracker != null}, OverwriteIfExist: {DoOverwrite == IfExists.Overwrite}");
}
}
/// <summary>
/// Creates a file in ADL store
/// </summary>
/// <param name="filePath">File Path</param>
/// <param name="accountName">Account name</param>
/// <param name="contents">Contents</param>
/// <param name="overwrite">True if we are overwriting the file if it exists, false if we want to fail if the file exists</param>
public void CreateFile(string filePath, string accountName, byte[] contents = null, IfExists overwrite = IfExists.Overwrite)
{
using (Stream createStream = AdlsClientFactory.GetAdlsClient(accountName, _context).CreateFile(filePath, overwrite))
{
if (contents != null)
{
createStream.Write(contents, 0, contents.Length);
}
}
}
// Testing purpose
internal static TransferStatus Download(string srcPath, string destPath, AdlsClient client, bool forceChunking, bool forceNotChunking, int numThreads = -1,
IProgress <TransferStatus> progressTracker = null, IfExists shouldOverwrite = IfExists.Overwrite, bool notRecurse = false, bool resume = false, bool egressTest = false, int egressBufferCapacity = 4 * 1024 * 1024, long chunkSize = ChunkSizeDefault)
{
if (forceChunking && forceNotChunking)
{
throw new ArgumentException("Both of them cant be true");
}
if (forceChunking)
{
SkipChunkingWeightThreshold = ChunkSizeDefault;
NumLargeFileThreshold = Int64.MaxValue;
}
else if (forceNotChunking)
{
SkipChunkingWeightThreshold = Int64.MaxValue;
}
return(new FileDownloader(srcPath, destPath, client, numThreads, shouldOverwrite, progressTracker, notRecurse, false, resume, default(CancellationToken), egressTest, egressBufferCapacity, chunkSize).RunTransfer());
}
/// <summary>
/// Download directory or file from remote server to local
/// </summary>
/// <param name="srcPath">Remote source path</param>
/// <param name="destPath">Local destination path</param>
/// <param name="client">ADLS client</param>
/// <param name="numThreads">Number of threads- if not passed will take default number of threads</param>
/// <param name="shouldOverwrite">Whether to overwrite or skip if the destination </param>
/// <param name="progressTracker">Progresstracker to track progress of file transfer</param>
/// <param name="notRecurse">If true then does enumeration till one level only, else will do recursive enumeration</param>
/// <param name="disableTransferLogging"></param>
/// <param name="resume">If true we are resuming a previously interrupted upload process</param>
/// <param name="cancelToken">Cancellation Token</param>
/// <param name="egressTest">Egress test when we do not write file to local file system</param>
/// <param name="egressBufferCapacity">Egress buffer size - Size of the read reuest from server</param>
/// <param name="chunkSize">Chunk Size used for chunking</param>
/// <returns>Transfer status of the download</returns>
internal static TransferStatus Download(string srcPath, string destPath, AdlsClient client, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool disableTransferLogging = false, bool resume = false, CancellationToken cancelToken = default(CancellationToken), bool egressTest = false, int egressBufferCapacity = 4 * 1024 * 1024, long chunkSize = ChunkSizeDefault)
{
if (!egressTest && string.IsNullOrWhiteSpace(destPath))
{
throw new ArgumentException(nameof(DestPath));
}
if (srcPath.EndsWith("/"))
{
srcPath = srcPath.Substring(0, srcPath.Length - 1);
}
if (destPath.EndsWith($"{Path.DirectorySeparatorChar}"))
{
destPath = destPath.Substring(0, destPath.Length - 1);
}
var downloader = new FileDownloader(srcPath, destPath, client, numThreads, shouldOverwrite, progressTracker,
notRecurse, disableTransferLogging, resume, cancelToken, egressTest, egressBufferCapacity, chunkSize);
return(downloader.RunTransfer());
}
/// <summary>
/// Reads data from memory stream and save it to local file
/// </summary>
/// <param name="srcPath">Remote source path</param>
/// <param name="destPath">Local destination path. It should always be a directory.</param>
/// <param name="numThreads">Not used</param>
/// <param name="shouldOverwrite">Whether to overwrite or skip if the destination exists</param>
/// <param name="progressTracker">Not used</param>
/// <param name="notRecurse">Not used</param>
/// <param name="resume">Not used</param>
/// <returns>Transfer status encapsulating the details of download</returns>
public override TransferStatus BulkDownload(string srcPath, string destPath, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool resume = false, CancellationToken cancelToken = default(CancellationToken))
{
var entry = _directoryEntries[srcPath].Entry;
if (entry.Type != DirectoryEntryType.FILE)
{
throw new ArgumentException("Currently not supported for folder");
}
using (Stream localStream = new FileStream(destPath, FileMode.Create, FileAccess.ReadWrite),
adlsStream = GetReadStream(srcPath))
{
byte[] buff = new byte[AdlsOutputStream.BufferCapacity];
while (true)
{
int bytesRead = adlsStream.Read(buff, 0, buff.Length);
if (bytesRead > 0)
{
localStream.Write(buff, 0, bytesRead);
}
else
{
break;
}
}
}
return(new TransferStatus());
}
/// <summary>
/// Bulk uploads file only. Reads a local file and maintains the memory stream for the entry
/// </summary>
/// <param name="srcPath">Local source path</param>
/// <param name="destPath">Remote destination path - It should always be a directory.</param>
/// <param name="numThreads">Not used</param>
/// <param name="shouldOverwrite">Whether to overwrite or skip if the destination exists</param>
/// <param name="progressTracker">Not used</param>
/// <param name="notRecurse">Not used</param>
/// <param name="resume">Not used</param>
/// <param name="isBinary">Not used</param>
/// <returns>Transfer Status encapsulating the details of upload</returns>
public override TransferStatus BulkUpload(string srcPath, string destPath, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool resume = false, bool isBinary = false, CancellationToken cancelToken = default(CancellationToken))
{
// Currently this is for a single file
if (!File.Exists(srcPath))
{
throw new ArgumentException("Currently not supported for folder");
}
using (Stream localStream = new FileStream(srcPath, FileMode.Open, FileAccess.Read),
adlsStream = CreateFile(destPath, shouldOverwrite))
{
byte[] buff = new byte[AdlsOutputStream.BufferCapacity];
while (true)
{
int bytesRead = localStream.Read(buff, 0, buff.Length);
if (bytesRead > 0)
{
adlsStream.Write(buff, 0, bytesRead);
}
else
{
break;
}
}
}
return(new TransferStatus());
}
/// <summary>
/// Creates an entry to the internal dictionary for the new file. The entry encapsulates AclStatus, DirectoryEntry and a memory stream
/// </summary>
/// <param name="filename">File name</param>
/// <param name="mode">If exists hether to overwrite or fail</param>
/// <param name="octalPermission">Permission string</param>
/// <param name="createParent">True if we create parent directories- currently has no effect</param>
/// <returns>Mock ADls output stream</returns>
public override AdlsOutputStream CreateFile(string filename, IfExists mode, string octalPermission = null,
bool createParent = true)
{
return(CreateFileAsync(filename, mode, octalPermission, createParent).GetAwaiter().GetResult());
}
/// <summary>
/// Creates an entry to the internal dictionary for the new file. The entry encapsulates AclStatus, DirectoryEntry and a memory stream
/// </summary>
/// <param name="filename">File name</param>
/// <param name="mode">If exists hether to overwrite or fail</param>
/// <param name="octalPermission">Permission string</param>
/// <param name="createParent">True if we create parent directories- currently has no effect</param>
/// <param name="cancelToken">Cancellation token</param>
/// <returns>Mock ADls output stream</returns>
public override async Task <AdlsOutputStream> CreateFileAsync(string filename, IfExists mode, string octalPermission = null,
bool createParent = true, CancellationToken cancelToken = default(CancellationToken))
{
return(await Task.Run(() =>
{
if (mode == IfExists.Fail && _directoryEntries.ContainsKey(filename))
{
throw new AdlsException("The file exists");
}
var metaData = CreateMetaData(filename, DirectoryEntryType.FILE, octalPermission);
_directoryEntries.Add(filename, metaData);
return new MockAdlsOutputStream(metaData.DataStream, metaData.Entry);
}, cancelToken));
}
protected FileTransferCommon(string srcPath, string destPath, AdlsClient client, int numThreads, IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool resume, bool ingressOrEgressTest, long chunkSize, string metaDataPath, CancellationToken cancelToken, string metaDataInfo = null)
{
if (string.IsNullOrWhiteSpace(srcPath))
{
throw new ArgumentException(nameof(srcPath));
}
SourcePath = srcPath;
DestPath = destPath;
Client = client;
// If ingress or egress test then simply overwrite destination
DoOverwrite = ingressOrEgressTest ? IfExists.Overwrite : doOverwrite;
ProgressTracker = progressTracker;
IngressOrEgressTest = ingressOrEgressTest;
NumConsumerThreads = numThreads <= 0 ? AdlsClient.DefaultNumThreads : numThreads;
ChunkSize = chunkSize;
NotRecurse = notRecurse;
metaDataInfo = string.IsNullOrEmpty(metaDataInfo) ? ChunkSize.ToString() : metaDataInfo + $",ChunkSize:{chunkSize},{(NotRecurse ? "NotRecurse" : "Recurse")}";
RecordedMetadata = new TransferLog(resume, metaDataPath, metaDataInfo);
Status = new TransferStatus();
ConsumerQueue = new PriorityQueueWrapper <BaseJob>();
CancelToken = cancelToken;
}
/// <summary>
/// Upload directory or file from local to remote
/// </summary>
/// <param name="srcPath">Local source path</param>
/// <param name="destPath">Remote destination path</param>
/// <param name="client">ADLS client</param>
/// <param name="numThreads">Number of threads- if not passed will take default number of threads</param>
/// <param name="shouldOverwrite">Whether to overwrite or skip if the destination </param>
/// <param name="progressTracker">Progresstracker to track progress of file transfer</param>
/// <param name="notRecurse"> If true then just does a enumeration in first level</param>
/// <param name="resume">If true we are resuming a previously interrupted upload process</param>
/// <param name="isBinary">If false then we want to upload at new line boundaries</param>
/// <param name="cancelToken">Cancellation Token</param>
/// <param name="ingressTest">True if we just want to test ingress</param>
/// <param name="chunkSize">Chunk Size used for chunking</param>
/// <returns>Transfer Status of the upload</returns>
internal static TransferStatus Upload(string srcPath, string destPath, AdlsClient client, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool resume = false, bool isBinary = false, CancellationToken cancelToken = default(CancellationToken), bool ingressTest = false, long chunkSize = ChunkSizeDefault)
{
if (string.IsNullOrWhiteSpace(destPath))
{
throw new ArgumentException(nameof(DestPath));
}
if (srcPath.EndsWith($"{Path.DirectorySeparatorChar}"))
{
srcPath = srcPath.Substring(0, srcPath.Length - 1);
}
// Check the destination path is not root
if (destPath.EndsWith("/") && !destPath.Equals("/"))
{
destPath = destPath.Substring(0, destPath.Length - 1);
}
var uploader = new FileUploader(srcPath, destPath, client, numThreads, shouldOverwrite, progressTracker, notRecurse, resume, isBinary, cancelToken, ingressTest, chunkSize);
return(uploader.RunTransfer());
}
