/// <summary> /// Creates an entry to the internal dictionary for the new file. The entry encapsulates AclStatus, DirectoryEntry and a memory stream /// </summary> /// <param name="filename">File name</param> /// <param name="mode">If exists hether to overwrite or fail</param> /// <param name="octalPermission">Permission string</param> /// <param name="createParent">True if we create parent directories- currently has no effect</param> /// <returns>Mock ADls output stream</returns> public override AdlsOutputStream CreateFile(string filename, IfExists mode, string octalPermission = null, bool createParent = true) { if (mode == IfExists.Fail && _directoryEntries.ContainsKey(filename)) { throw new AdlsException("The file exists"); } var metaData = CreateMetaData(filename, DirectoryEntryType.FILE, octalPermission); _directoryEntries.Add(filename, metaData); return(new MockAdlsOutputStream(metaData.DataStream)); }
private FileUploader(string srcPath, string destPath, AdlsClient client, int numThreads, IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool resume, bool isBinary, CancellationToken cancelToken, bool ingressTest, long chunkSize) : base(srcPath, destPath, client, numThreads, doOverwrite, progressTracker, notRecurse, resume, ingressTest, chunkSize, Path.Combine(Path.GetTempPath(), ".adl", "Upload", GetTransferLogFileName(srcPath, destPath, Path.DirectorySeparatorChar, '/')), cancelToken, $"binary:{isBinary}") { // If not recurse then we will have one thread and ProducerFirstPass logic loop will run only once NumProducerThreads = NotRecurse ? 1 : NumProducerThreadsFirstPass; UploaderProducerQueue = new QueueWrapper <DirectoryInfo>(NumProducerThreads); if (FileTransferLog.IsDebugEnabled) { FileTransferLog.Debug($"FileTransfer.Uploader, Src: {SourcePath}, Dest: {DestPath}, Threads: {NumConsumerThreads}, TrackingProgress: {ProgressTracker != null}, OverwriteIfExist: {DoOverwrite == IfExists.Overwrite}"); } _isBinary = isBinary; _encodeType = Encoding.UTF8; }
private FileDownloader(string srcPath, string destPath, AdlsClient client, int numThreads, IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool disableTransferLogging, bool resume, CancellationToken cancelToken, bool egressTest, int egressBufferCapacity, long chunkSize) : base(srcPath, destPath, client, numThreads, doOverwrite, progressTracker, notRecurse, disableTransferLogging, resume, egressTest, chunkSize, Path.Combine(Path.GetTempPath(), ".adl", "Download", GetTransferLogFileName(client.AccountFQDN, srcPath, destPath, '/', Path.DirectorySeparatorChar)), cancelToken) { EgressBufferCapacity = egressBufferCapacity; // If not recurse then we will have one thread and ProducerFirstPass logic loop will run only once NumProducerThreads = notRecurse ? 1 : NumProducerThreadsFirstPass; DownloaderProducerQueue = new QueueWrapper <DirectoryEntry>(NumProducerThreads); DownloaderList = new List <DirectoryEntry>(DownloaderListCapacity); if (FileTransferLog.IsDebugEnabled) { FileTransferLog.Debug($"FileTransfer.Downloader, Src: {SourcePath}, Dest: {DestPath}, Threads: {NumConsumerThreads}, TrackingProgress: {ProgressTracker != null}, OverwriteIfExist: {DoOverwrite == IfExists.Overwrite}"); } }
/// <summary> /// Creates a file in ADL store /// </summary> /// <param name="filePath">File Path</param> /// <param name="accountName">Account name</param> /// <param name="contents">Contents</param> /// <param name="overwrite">True if we are overwriting the file if it exists, false if we want to fail if the file exists</param> public void CreateFile(string filePath, string accountName, byte[] contents = null, IfExists overwrite = IfExists.Overwrite) { using (Stream createStream = AdlsClientFactory.GetAdlsClient(accountName, _context).CreateFile(filePath, overwrite)) { if (contents != null) { createStream.Write(contents, 0, contents.Length); } } }
// Testing purpose internal static TransferStatus Download(string srcPath, string destPath, AdlsClient client, bool forceChunking, bool forceNotChunking, int numThreads = -1, IProgress <TransferStatus> progressTracker = null, IfExists shouldOverwrite = IfExists.Overwrite, bool notRecurse = false, bool resume = false, bool egressTest = false, int egressBufferCapacity = 4 * 1024 * 1024, long chunkSize = ChunkSizeDefault) { if (forceChunking && forceNotChunking) { throw new ArgumentException("Both of them cant be true"); } if (forceChunking) { SkipChunkingWeightThreshold = ChunkSizeDefault; NumLargeFileThreshold = Int64.MaxValue; } else if (forceNotChunking) { SkipChunkingWeightThreshold = Int64.MaxValue; } return(new FileDownloader(srcPath, destPath, client, numThreads, shouldOverwrite, progressTracker, notRecurse, false, resume, default(CancellationToken), egressTest, egressBufferCapacity, chunkSize).RunTransfer()); }
/// <summary> /// Download directory or file from remote server to local /// </summary> /// <param name="srcPath">Remote source path</param> /// <param name="destPath">Local destination path</param> /// <param name="client">ADLS client</param> /// <param name="numThreads">Number of threads- if not passed will take default number of threads</param> /// <param name="shouldOverwrite">Whether to overwrite or skip if the destination </param> /// <param name="progressTracker">Progresstracker to track progress of file transfer</param> /// <param name="notRecurse">If true then does enumeration till one level only, else will do recursive enumeration</param> /// <param name="disableTransferLogging"></param> /// <param name="resume">If true we are resuming a previously interrupted upload process</param> /// <param name="cancelToken">Cancellation Token</param> /// <param name="egressTest">Egress test when we do not write file to local file system</param> /// <param name="egressBufferCapacity">Egress buffer size - Size of the read reuest from server</param> /// <param name="chunkSize">Chunk Size used for chunking</param> /// <returns>Transfer status of the download</returns> internal static TransferStatus Download(string srcPath, string destPath, AdlsClient client, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool disableTransferLogging = false, bool resume = false, CancellationToken cancelToken = default(CancellationToken), bool egressTest = false, int egressBufferCapacity = 4 * 1024 * 1024, long chunkSize = ChunkSizeDefault) { if (!egressTest && string.IsNullOrWhiteSpace(destPath)) { throw new ArgumentException(nameof(DestPath)); } if (srcPath.EndsWith("/")) { srcPath = srcPath.Substring(0, srcPath.Length - 1); } if (destPath.EndsWith($"{Path.DirectorySeparatorChar}")) { destPath = destPath.Substring(0, destPath.Length - 1); } var downloader = new FileDownloader(srcPath, destPath, client, numThreads, shouldOverwrite, progressTracker, notRecurse, disableTransferLogging, resume, cancelToken, egressTest, egressBufferCapacity, chunkSize); return(downloader.RunTransfer()); }
/// <summary> /// Reads data from memory stream and save it to local file /// </summary> /// <param name="srcPath">Remote source path</param> /// <param name="destPath">Local destination path. It should always be a directory.</param> /// <param name="numThreads">Not used</param> /// <param name="shouldOverwrite">Whether to overwrite or skip if the destination exists</param> /// <param name="progressTracker">Not used</param> /// <param name="notRecurse">Not used</param> /// <param name="resume">Not used</param> /// <returns>Transfer status encapsulating the details of download</returns> public override TransferStatus BulkDownload(string srcPath, string destPath, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool resume = false, CancellationToken cancelToken = default(CancellationToken)) { var entry = _directoryEntries[srcPath].Entry; if (entry.Type != DirectoryEntryType.FILE) { throw new ArgumentException("Currently not supported for folder"); } using (Stream localStream = new FileStream(destPath, FileMode.Create, FileAccess.ReadWrite), adlsStream = GetReadStream(srcPath)) { byte[] buff = new byte[AdlsOutputStream.BufferCapacity]; while (true) { int bytesRead = adlsStream.Read(buff, 0, buff.Length); if (bytesRead > 0) { localStream.Write(buff, 0, bytesRead); } else { break; } } } return(new TransferStatus()); }
/// <summary> /// Bulk uploads file only. Reads a local file and maintains the memory stream for the entry /// </summary> /// <param name="srcPath">Local source path</param> /// <param name="destPath">Remote destination path - It should always be a directory.</param> /// <param name="numThreads">Not used</param> /// <param name="shouldOverwrite">Whether to overwrite or skip if the destination exists</param> /// <param name="progressTracker">Not used</param> /// <param name="notRecurse">Not used</param> /// <param name="resume">Not used</param> /// <param name="isBinary">Not used</param> /// <returns>Transfer Status encapsulating the details of upload</returns> public override TransferStatus BulkUpload(string srcPath, string destPath, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool resume = false, bool isBinary = false, CancellationToken cancelToken = default(CancellationToken)) { // Currently this is for a single file if (!File.Exists(srcPath)) { throw new ArgumentException("Currently not supported for folder"); } using (Stream localStream = new FileStream(srcPath, FileMode.Open, FileAccess.Read), adlsStream = CreateFile(destPath, shouldOverwrite)) { byte[] buff = new byte[AdlsOutputStream.BufferCapacity]; while (true) { int bytesRead = localStream.Read(buff, 0, buff.Length); if (bytesRead > 0) { adlsStream.Write(buff, 0, bytesRead); } else { break; } } } return(new TransferStatus()); }
/// <summary> /// Creates an entry to the internal dictionary for the new file. The entry encapsulates AclStatus, DirectoryEntry and a memory stream /// </summary> /// <param name="filename">File name</param> /// <param name="mode">If exists hether to overwrite or fail</param> /// <param name="octalPermission">Permission string</param> /// <param name="createParent">True if we create parent directories- currently has no effect</param> /// <returns>Mock ADls output stream</returns> public override AdlsOutputStream CreateFile(string filename, IfExists mode, string octalPermission = null, bool createParent = true) { return(CreateFileAsync(filename, mode, octalPermission, createParent).GetAwaiter().GetResult()); }
/// <summary> /// Creates an entry to the internal dictionary for the new file. The entry encapsulates AclStatus, DirectoryEntry and a memory stream /// </summary> /// <param name="filename">File name</param> /// <param name="mode">If exists hether to overwrite or fail</param> /// <param name="octalPermission">Permission string</param> /// <param name="createParent">True if we create parent directories- currently has no effect</param> /// <param name="cancelToken">Cancellation token</param> /// <returns>Mock ADls output stream</returns> public override async Task <AdlsOutputStream> CreateFileAsync(string filename, IfExists mode, string octalPermission = null, bool createParent = true, CancellationToken cancelToken = default(CancellationToken)) { return(await Task.Run(() => { if (mode == IfExists.Fail && _directoryEntries.ContainsKey(filename)) { throw new AdlsException("The file exists"); } var metaData = CreateMetaData(filename, DirectoryEntryType.FILE, octalPermission); _directoryEntries.Add(filename, metaData); return new MockAdlsOutputStream(metaData.DataStream, metaData.Entry); }, cancelToken)); }
protected FileTransferCommon(string srcPath, string destPath, AdlsClient client, int numThreads, IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool resume, bool ingressOrEgressTest, long chunkSize, string metaDataPath, CancellationToken cancelToken, string metaDataInfo = null) { if (string.IsNullOrWhiteSpace(srcPath)) { throw new ArgumentException(nameof(srcPath)); } SourcePath = srcPath; DestPath = destPath; Client = client; // If ingress or egress test then simply overwrite destination DoOverwrite = ingressOrEgressTest ? IfExists.Overwrite : doOverwrite; ProgressTracker = progressTracker; IngressOrEgressTest = ingressOrEgressTest; NumConsumerThreads = numThreads <= 0 ? AdlsClient.DefaultNumThreads : numThreads; ChunkSize = chunkSize; NotRecurse = notRecurse; metaDataInfo = string.IsNullOrEmpty(metaDataInfo) ? ChunkSize.ToString() : metaDataInfo + $",ChunkSize:{chunkSize},{(NotRecurse ? "NotRecurse" : "Recurse")}"; RecordedMetadata = new TransferLog(resume, metaDataPath, metaDataInfo); Status = new TransferStatus(); ConsumerQueue = new PriorityQueueWrapper <BaseJob>(); CancelToken = cancelToken; }
/// <summary> /// Upload directory or file from local to remote /// </summary> /// <param name="srcPath">Local source path</param> /// <param name="destPath">Remote destination path</param> /// <param name="client">ADLS client</param> /// <param name="numThreads">Number of threads- if not passed will take default number of threads</param> /// <param name="shouldOverwrite">Whether to overwrite or skip if the destination </param> /// <param name="progressTracker">Progresstracker to track progress of file transfer</param> /// <param name="notRecurse"> If true then just does a enumeration in first level</param> /// <param name="resume">If true we are resuming a previously interrupted upload process</param> /// <param name="isBinary">If false then we want to upload at new line boundaries</param> /// <param name="cancelToken">Cancellation Token</param> /// <param name="ingressTest">True if we just want to test ingress</param> /// <param name="chunkSize">Chunk Size used for chunking</param> /// <returns>Transfer Status of the upload</returns> internal static TransferStatus Upload(string srcPath, string destPath, AdlsClient client, int numThreads = -1, IfExists shouldOverwrite = IfExists.Overwrite, IProgress <TransferStatus> progressTracker = null, bool notRecurse = false, bool resume = false, bool isBinary = false, CancellationToken cancelToken = default(CancellationToken), bool ingressTest = false, long chunkSize = ChunkSizeDefault) { if (string.IsNullOrWhiteSpace(destPath)) { throw new ArgumentException(nameof(DestPath)); } if (srcPath.EndsWith($"{Path.DirectorySeparatorChar}")) { srcPath = srcPath.Substring(0, srcPath.Length - 1); } // Check the destination path is not root if (destPath.EndsWith("/") && !destPath.Equals("/")) { destPath = destPath.Substring(0, destPath.Length - 1); } var uploader = new FileUploader(srcPath, destPath, client, numThreads, shouldOverwrite, progressTracker, notRecurse, resume, isBinary, cancelToken, ingressTest, chunkSize); return(uploader.RunTransfer()); }