/// <summary>
/// Gets the rows from head or tail
/// </summary>
/// <param name="streamPath">File path</param>
/// <param name="accountName">Account name</param>
/// <param name="numRows">Number of rows user wants to see</param>
/// <param name="encoding">Encoding</param>
/// <param name="reverse">True if we are reading from tail else </param>
/// <returns>List of lines</returns>
public IEnumerable <string> GetStreamRows(string streamPath, string accountName, int numRows, Encoding encoding, bool reverse = false)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
if (!reverse)
{
return(GetStreamRowsHead(client, streamPath, numRows, encoding));
}
var toReturn = new LinkedList <string>();
using (Stream readerAdl = client.GetReadStream(streamPath))
{
// read data 4MB at a time
// when reading backwards, this may change to ensure that we don't re-read data as we approach the beginning of the file.
var dataPerRead = 4 * 1024 * 1024;
bool doneAfterRead = false;
var fileLength = readerAdl.Length;
if (fileLength <= dataPerRead)
{
doneAfterRead = true;
}
long initialOffset = fileLength - dataPerRead;
// while we haven't finished loading all the rows, keep grabbing content
var readRows = 0;
byte[] buffer = new byte[dataPerRead];
do
{
if (initialOffset < 0)
{
initialOffset = 0;
doneAfterRead = true;
}
readerAdl.Seek(initialOffset, SeekOrigin.Begin);
int dataActuallyRead = readerAdl.Read(buffer, 0, dataPerRead);
// Reads the lines, updates the number of lines and returns the last poisiotn of \r or \n
int newLineOffset = ReadNewLinesReverse(buffer, dataActuallyRead, encoding, numRows, ref toReturn, ref readRows);
if (newLineOffset != -1)
{
initialOffset += newLineOffset;
initialOffset -= dataPerRead;
}
// this denotes that we did not get as many rows as desired but we ran out of file.
if (doneAfterRead)
{
break;
}
} while (readRows < numRows);
}
return(toReturn);
}
/// <summary>
/// Get items in trash matching query string
/// </summary>
/// <param name="accountName">Account name</param>
/// <param name="filter">Query to match items in trash</param>
/// <param name="count">Minimum number of entries to search for</param>
/// <param name="cmdletCancellationToken">CancellationToken</param>
public IEnumerable <TrashEntry> EnumerateDeletedItems(string accountName, string filter, int count, CancellationToken cmdletCancellationToken = default(CancellationToken))
{
IEnumerable <TrashEntry> result = null;
Task enumerateTask = null;
bool completed = false;
object syncPrimitive = new object();
var progressTracker = new Progress <EnumerateDeletedItemsProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
// TODO: Update job progress here
lock (syncPrimitive)
{
if (e.NumFound >= count || String.IsNullOrEmpty(e.NextListAfter))
{
completed = true;
}
Monitor.Pulse(syncPrimitive);
}
};
enumerateTask = Task.Run(() =>
{
result = AdlsClientFactory.GetAdlsClient(accountName, _context).EnumerateDeletedItems(filter, "", count, progressTracker, cmdletCancellationToken);
}, cmdletCancellationToken);
// Keep printing (?) the updates returned by successive calls by the SDK to the backend, until the whole query completes
while (true)
{
lock (syncPrimitive)
{
Monitor.Wait(syncPrimitive);
if (completed)
{
break;
}
}
}
WaitForTask(enumerateTask, cmdletCancellationToken);
return(result);
}
/// <summary>
/// Get file properties
/// </summary>
/// <param name="accountName">Account name</param>
/// <param name="path">Path</param>
/// <param name="getAclUsage">Get Acl usage</param>
/// <param name="dumpFileName">Dump file name</param>
/// <param name="getDiskUsage">GetDisk usage</param>
/// <param name="saveToLocal">Save to local</param>
/// <param name="numThreads">Number of threads</param>
/// <param name="displayFiles">Display files</param>
/// <param name="hideConsistentAcl">Hide consistent Acl</param>
/// <param name="maxDepth">Maximum depth</param>
/// <param name="cmdlet">ExportFileProperties Commandlet</param>
/// <param name="cmdletCancellationToken">CancellationToken</param>
public void GetFileProperties(string accountName, string path, bool getAclUsage, string dumpFileName, bool getDiskUsage, bool saveToLocal, int numThreads, bool displayFiles, bool hideConsistentAcl, long maxDepth, Cmdlet cmdlet, CancellationToken cmdletCancellationToken)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
Task exportTask = Task.Run(() => client
.GetFileProperties(
path, getAclUsage, dumpFileName,
getDiskUsage, saveToLocal, numThreads, displayFiles, hideConsistentAcl, maxDepth,
cmdletCancellationToken), cmdletCancellationToken);
if (_isDebugEnabled)
{
TrackTaskProgress(exportTask, cmdlet, null, cmdletCancellationToken);
}
else
{
WaitForTask(exportTask, cmdletCancellationToken);
}
}
/// <summary>
/// Checks if file or folder exists and also returns the type of the path
/// </summary>
/// <param name="path">Full path of file or directory</param>
/// <param name="accountName">Account name</param>
/// <param name="itemType">Type of the directory entry</param>
/// <returns>True of the path exists else false</returns>
public bool TestFileOrFolderExistence(string path, string accountName, out DirectoryEntryType itemType)
{
try
{
var entry = AdlsClientFactory.GetAdlsClient(accountName, _context).GetDirectoryEntry(path);
itemType = entry.Type;
return(true);
}
catch (AdlsException e)
{
if (e.HttpStatus == HttpStatusCode.NotFound)
{
itemType = DirectoryEntryType.FILE;
return(false);
}
throw e;
}
}
/// <summary>
/// Sets the expiry time of the file. If expiry option is not specified then tries to guess the option based on timeToSet value.
/// </summary>
/// <param name="path">Path</param>
/// <param name="accountName">Account name</param>
/// <param name="timeToSet">Time to set</param>
/// <param name="exop">ExpiryOption</param>
/// <returns>FileStatus after setting the expiry</returns>
public DirectoryEntry SetExpiry(string path, string accountName, long timeToSet, ExpiryOption?exop = null)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
if (exop == null)
{
if (timeToSet <= 0 || timeToSet >= NeverExpireValue)
{
exop = ExpiryOption.NeverExpire;
}
else
{
exop = ExpiryOption.Absolute;
}
}
client.SetExpiryTime(path, exop.Value, timeToSet);
return(GetFileStatus(path, accountName));
}
/// <summary>
/// Get items in trash matching query string
/// </summary>
/// <param name="accountName">Account name</param>
/// <param name="filter">Query to match items in trash</param>
/// <param name="count">Minimum number of entries to search for</param>
/// <param name="cmdlet"></param>
/// <param name="cmdletCancellationToken">CancellationToken</param>
public IEnumerable <TrashEntry> EnumerateDeletedItems(string accountName, string filter, int count, Cmdlet cmdlet, CancellationToken cmdletCancellationToken = default(CancellationToken))
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
if (_isDebugEnabled)
{
IEnumerable <TrashEntry> result = null;
// Api call below consists of multiple rest calls so multiple debug statements will be posted
// so since we want to the debug lines to be updated while the command runs, we have to flush the debug statements in queue and thats why we want to do it this way
var enumerateTask = Task.Run(() => {
result = client.EnumerateDeletedItems(filter, "", count, null, cmdletCancellationToken);
}, cmdletCancellationToken);
TrackTaskProgress(enumerateTask, cmdlet, null, cmdletCancellationToken);
return(result);
}
else
{
return(client.EnumerateDeletedItems(filter, "", count, null, cmdletCancellationToken));
}
}
/// <summary>
/// Returns the ADL read stream
/// </summary>
/// <param name="filePath">File Path</param>
/// <param name="accountName">Account name</param>
/// <returns>AdlReadStream</returns>
public Stream ReadFromFile(string filePath, string accountName)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).GetReadStream(filePath));
}
/// <summary>
/// Renames a file or directory
/// </summary>
/// <param name="sourcePath">Source Path</param>
/// <param name="accountName">Account name</param>
/// <param name="destinationPath">Destination path</param>
/// <returns>True if rename is successful else fale</returns>
public bool RenameFileOrDirectory(string sourcePath, string accountName, string destinationPath)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).Rename(sourcePath, destinationPath));
}
/// <summary>
///
/// </summary>
/// <param name="path"></param>
/// <param name="accountName"></param>
/// <param name="aclToSet"></param>
/// <param name="aclChangeType"></param>
/// <param name="concurrency"></param>
public AclProcessorStats ChangeAclRecursively(string path, string accountName, List <AclEntry> aclToSet,
RequestedAclType aclChangeType, int concurrency = -1)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).ChangeAcl(path, aclToSet, aclChangeType, concurrency));
}
/// <summary>
/// Performs the bulk copy and tracks the progress
/// </summary>
/// <param name="destinationFolderPath">Destination folder path</param>
/// <param name="accountName">Account name</param>
/// <param name="sourcePath">Source folder or file path</param>
/// <param name="cmdletCancellationToken">Commandlet cancellation token</param>
/// <param name="fileThreadCount">PerFileThreadCount</param>
/// <param name="recursive">If true then Enumeration of the subdirectories are done recursively</param>
/// <param name="overwrite">True if we want to overwrite existing destinations</param>
/// <param name="resume">Indicates that the file(s) being copied are a continuation of a previous file transfer</param>
/// <param name="isDownload">True if it is download else upload</param>
/// <param name="cmdletRunningRequest">Current Commandlet</param>
/// <param name="isBinary">Indicates that the file(s) being uploaded should be copied with no concern for new line preservation across appends</param>
public void BulkCopy(string destinationFolderPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken,
int fileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool isDownload = false,
Cmdlet cmdletRunningRequest = null, bool isBinary = false)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
var progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Enumerating the source and preparing the copy.",
sourcePath, recursive ? " recursively" : string.Empty), "Copy in progress...")
{
PercentComplete = 0
};
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <TransferStatus>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * (e.ChunksTransfered + e.FilesTransfered + e.DirectoriesTransferred) / (e.TotalChunksToTransfer + e.TotalFilesToTransfer + e.TotalDirectoriesToTransfer) * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
progress.Activity =
$"Copying Folder: {sourcePath}{(recursive ? " recursively" : string.Empty)}. Bytes remaining: {e.TotalSizeToTransfer - e.SizeTransfered}{(e.TotalChunksToTransfer > 0 ? $", Chunks remaining: {e.TotalChunksToTransfer - e.ChunksTransfered}" : "")}{(e.TotalNonChunkedFileToTransfer > 0 ? $", Non-chunked files remaining: {e.TotalNonChunkedFileToTransfer - e.NonChunkedFileTransferred}" : "")}" +
$"{(e.TotalDirectoriesToTransfer > 0 ? $", Directories remaining: {e.TotalDirectoriesToTransfer - e.DirectoriesTransferred}" : "")}.";
}
};
TransferStatus status = null;
Task transferTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
if (isDownload)
{
status = client.BulkDownload(sourcePath, destinationFolderPath, fileThreadCount,
overwrite ? IfExists.Overwrite : IfExists.Fail, progressTracker, !recursive, resume, cmdletCancellationToken);
}
else
{
status = client.BulkUpload(sourcePath, destinationFolderPath, fileThreadCount,
overwrite ? IfExists.Overwrite : IfExists.Fail, progressTracker, !recursive, resume, isBinary, cmdletCancellationToken);
}
}, cmdletCancellationToken);
TrackTaskProgress(transferTask, cmdletRunningRequest, progress, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
if (status != null && cmdletRunningRequest != null)
{
foreach (var failedEntry in status.EntriesFailed)
{
cmdletRunningRequest.WriteObject($"FailedTransfer: {failedEntry.EntryName} {failedEntry.Errors}");
}
}
}
}
/// <summary>
/// Concats the files
/// </summary>
/// <param name="destinationPath">Destination path</param>
/// <param name="accountName">Account name</param>
/// <param name="filesToConcatenate">List of files to concatenate</param>
public void ConcatenateFiles(string destinationPath, string accountName, List <string> filesToConcatenate)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).ConcatenateFiles(destinationPath, filesToConcatenate);
}
/// <summary>
/// Obtains the content summary recursively
/// </summary>
/// <param name="path">File Path</param>
/// <param name="accountName">Account name</param>
/// <param name="numThreads">Concurrency</param>
/// <param name="cancelToken">Cancelation token</param>
/// <returns></returns>
public ContentSummary GetContentSummary(string path, string accountName, int numThreads, CancellationToken cancelToken)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).GetContentSummary(path, numThreads, cancelToken));
}
/// <summary>
/// Add specific Acl entries
/// </summary>
/// <param name="path">Path</param>
/// <param name="accountName">Account</param>
/// <param name="aclToModify">Acl list specifying acls to modify</param>
public void ModifyAcl(string path, string accountName, List <AclEntry> aclToModify)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).ModifyAclEntries(path, aclToModify);
}
/// <summary>
/// Changes Acl recursively
/// </summary>
/// <param name="path">Input path</param>
/// <param name="accountName">Account name</param>
/// <param name="aclToSet">List of acl to set</param>
/// <param name="aclChangeType">Type of al change- Modify, Set, Remove</param>
/// <param name="concurrency">Concurrency- number of parallel operations</param>
/// <param name="aclCmdlet">Cmdlet for acl change. This is only for printing progress. If passed null, then no progress tracking is done</param>
/// <param name="trackProgress"></param>
/// <param name="cmdletCancellationToken">Cancellationtoken for cmdlet</param>
public AclProcessorStats ChangeAclRecursively(string path, string accountName, List <AclEntry> aclToSet,
RequestedAclType aclChangeType, int concurrency, Cmdlet aclCmdlet, bool trackProgress, CancellationToken cmdletCancellationToken)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
// Currently mockadlsclient signature is different, once that gets fixed, we can remove this
if (client.GetType() != typeof(MockAdlsClient))
{
System.Progress <AclProcessorStats> progressTracker = null;
ProgressRecord progress = null;
// If passing null, then we do not want progreess tracking
if (trackProgress)
{
progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000),
string.Format($"Recursive acl change for path {path}"),
$"Type of Acl Change: {aclChangeType}")
{
PercentComplete = 0
};
// On update from the Data Lake store uploader, capture the progress.
progressTracker = new System.Progress <AclProcessorStats>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = 0;
progress.Activity =
$"Files enumerated: {e.FilesProcessed} Directories enumerated:{e.DirectoryProcessed}";
}
};
}
AclProcessorStats status = null;
Task aclTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
status = client.ChangeAcl(path, aclToSet, aclChangeType, concurrency, progressTracker,
cmdletCancellationToken);
}, cmdletCancellationToken);
if (trackProgress || _isDebugEnabled)
{
TrackTaskProgress(aclTask, aclCmdlet, progress, cmdletCancellationToken);
if (trackProgress && !cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, aclCmdlet);
}
}
else
{
WaitForTask(aclTask, cmdletCancellationToken);
}
return(status);
}
return(client.ChangeAcl(path, aclToSet, aclChangeType, concurrency));
}
/// <summary>
/// Sets specific Acl entries
/// </summary>
/// <param name="path">Path</param>
/// <param name="accountName">Account</param>
/// <param name="aclToSet">Acl list specifying acls to set</param>
public void SetAcl(string path, string accountName, List <AclEntry> aclToSet)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).SetAcl(path, aclToSet);
}
/// <summary>
/// Sets the owner of the path
/// </summary>
/// <param name="path">Path name</param>
/// <param name="accountName">Account name</param>
/// <param name="owner">Owner</param>
/// <param name="group">Group</param>
public void SetOwner(string path, string accountName, string owner, string group)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).SetOwner(path, owner, group);
}
/// <summary>
/// Sets the permission
/// </summary>
/// <param name="path">Path name</param>
/// <param name="accountName">Account name</param>
/// <param name="permissionsToSet">Permission to set</param>
public void SetPermission(string path, string accountName, string permissionsToSet)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).SetPermission(path, permissionsToSet);
}
/// <summary>
/// Get file properties
/// </summary>
/// <param name="accountName">Account name</param>
/// <param name="path">Path</param>
/// <param name="getAclUsage">Get Acl usage</param>
/// <param name="dumpFileName">Dump file name</param>
/// <param name="getDiskUsage">GetDisk usage</param>
/// <param name="saveToLocal">Save to local</param>
/// <param name="numThreads">Number of threads</param>
/// <param name="displayFiles">Display files</param>
/// <param name="hideConsistentAcl">Hide consistent Acl</param>
/// <param name="maxDepth">Maximum depth</param>
public void GetFileProperties(string accountName, string path, bool getAclUsage, string dumpFileName, bool getDiskUsage, bool saveToLocal, int numThreads, bool displayFiles, bool hideConsistentAcl, long maxDepth)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).GetFileProperties(path, getAclUsage, dumpFileName,
getDiskUsage, saveToLocal, numThreads, displayFiles, hideConsistentAcl, maxDepth);
}
/// <summary>
/// Returns an ienumerable of directoryentry
/// </summary>
/// <param name="folderPath">Full directory path</param>
/// <param name="accountName">Account name</param>
/// <returns>IEnumerable of directory entry</returns>
public IEnumerable <DirectoryEntry> GetFileStatuses(string folderPath, string accountName)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).EnumerateDirectory(folderPath));
}
/// <summary>
/// Gets the properties of the file or directory
/// </summary>
/// <param name="filePath">Full path of the file or directory</param>
/// <param name="accountName">Account Name</param>
/// <returns>Instance encapsulating the properties of the path</returns>
public DirectoryEntry GetFileStatus(string filePath, string accountName)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).GetDirectoryEntry(filePath));
}
/// <summary>
/// Removes specific Acl entries
/// </summary>
/// <param name="path">Path</param>
/// <param name="accountName">Account</param>
/// <param name="aclsToRemove">Acl list specifying acls to remove</param>
public void RemoveAclEntries(string path, string accountName, List <AclEntry> aclsToRemove)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).RemoveAclEntries(path, aclsToRemove);
}
/// <summary>
/// Deletes the file or folder.
/// </summary>
/// <param name="path">Path</param>
/// <param name="accountName">Account name</param>
/// <param name="isRecursive">For directories if we want recursive delete</param>
/// <returns>True if the delete is successful else false</returns>
public bool DeleteFileOrFolder(string path, string accountName, bool isRecursive)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
return(isRecursive ? client.DeleteRecursive(path) : client.Delete(path));
}
/// <summary>
/// Obtains the content summary recursively
/// </summary>
/// <param name="path">File Path</param>
/// <param name="accountName">Account name</param>
/// <returns></returns>
public ContentSummary GetContentSummary(string path, string accountName)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).GetContentSummary(path));
}
/// <summary>
/// Creates a directory in the store
/// </summary>
/// <param name="dirPath">Directory Path</param>
/// <param name="accountName">Account name</param>
/// <returns>True if the directory is created successfully else false</returns>
public bool CreateDirectory(string dirPath, string accountName)
{
var boolean = AdlsClientFactory.GetAdlsClient(accountName, _context).CreateDirectory(dirPath);
return(boolean);
}
/// <summary>
/// Retrieves the Acl status of the path
/// </summary>
/// <param name="filePath">File or directory Path</param>
/// <param name="accountName">Account name</param>
/// <returns>Acl Status</returns>
public AclStatus GetAclStatus(string filePath, string accountName)
{
return(GetFullAcl(AdlsClientFactory.GetAdlsClient(accountName, _context).GetAclStatus(filePath)));
}
/// <summary>
/// Remove all the ACLs
/// </summary>
/// <param name="path">Path name</param>
/// <param name="accountName">Account name</param>
public void RemoveAcl(string path, string accountName)
{
AdlsClientFactory.GetAdlsClient(accountName, _context).RemoveAllAcls(path);
}
/// <summary>
/// Restore a stream or directory from trash to user space. This is a synchronous operation.
/// Not threadsafe when Restore is called for same path from different threads.
/// </summary>
/// <param name="accountName">Account name</param>
/// <param name="path">The trash directory path in enumeratedeleteditems response</param>
/// <param name="destination">Path to where the entry should be restored</param>
/// <param name="type">Type of the entry which is being restored. "file" or "folder"</param>
/// <param name="restoreAction">Action to take during destination name conflicts - "overwrite" or "copy"</param>
/// <param name="cmdletCancellationToken">CancellationToken</param>
public void RestoreDeletedItem(string accountName, string path, string destination, string type, string restoreAction, CancellationToken cmdletCancellationToken = default(CancellationToken))
{
AdlsClientFactory.GetAdlsClient(accountName, _context).RestoreDeletedItems(path, destination, type, restoreAction, cmdletCancellationToken);
}