private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, IProgress <AclProcessorStats> aclStatusTracker, CancellationToken cancelToken, bool verify = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount <= 0 ? AdlsClient.DefaultNumThreads: threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
if (aclEntries == null || aclEntries.Count == 0)
{
throw new ArgumentException("Input acl is null or empty");
}
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
if (FileAclEntries.Count == 0 && AclLog.IsDebugEnabled)
{
AclLog.Debug("AclEntries for file are empty so input acl must be containing default acls");
}
Type = type;
_isVerify = verify;
_aclStatusTracker = aclStatusTracker;
_cancelToken = cancelToken;
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":",AclEntries)}{(_isVerify?", RunInVerifyMode":string.Empty)}");
}
}
private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, IProgress <AclProcessorStats> aclStatusTracker, CancellationToken cancelToken, bool verify = false, string verifyFile = null, bool ignoreVerifyTimeErrors = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount <= 0 ? AdlsClient.DefaultNumThreads : threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
if (aclEntries == null || aclEntries.Count == 0)
{
throw new ArgumentException("Input acl is null or empty");
}
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
if (FileAclEntries.Count == 0 && AclLog.IsDebugEnabled)
{
AclLog.Debug("AclEntries for file are empty so input acl must be containing default acls");
}
Type = type;
_isVerify = verify;
_aclStatusTracker = aclStatusTracker;
_cancelToken = cancelToken;
// If verify file is passed we have to setup a thread and a filestream to write to the file
if (verify && !string.IsNullOrEmpty(verifyFile))
{
_ignoreVerifyTimeErrors = ignoreVerifyTimeErrors;
_incorrectVerifyFile = verifyFile;
_incorrectFileList = new QueueWrapper <string>(-1);
Utils.CreateParentDirectory(_incorrectVerifyFile);
_incorrectVerifyFileStream = new StreamWriter(new FileStream(_incorrectVerifyFile, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
AutoFlush = true
};
}
_linkPaths = new ConcurrentBag <string>();
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":", AclEntries)}{(_isVerify ? ", RunInVerifyMode" : string.Empty)}");
}
}
private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, bool verify = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount < 0 ? AdlsClient.DefaultNumThreads: threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
Type = type;
_isVerify = verify;
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":",AclEntries)}{(_isVerify?", RunInVerifyMode":string.Empty)}");
}
}
/// <summary>
/// Internal test Api to verify Acl Processor. Runs Acl verifier and returns number of files and directories processed correctly.
/// </summary>
/// <param name="path">Root path from where the Acl recursive verifier will start</param>
/// <param name="client">ADLS Client</param>
/// <param name="aclEntries">Acl Entries to verify</param>
/// <param name="type">Type of Acl Job: Acl modify verify or Acl set verify or acl remove verify</param>
/// <param name="threadCount">Custom number of threads</param>
/// <returns></returns>
internal static AclProcessorStats RunAclVerifier(string path, AdlsClient client, List <AclEntry> aclEntries,
RequestedAclType type, int threadCount = -1)
{
return(new AclProcessor(path, client, aclEntries, type, threadCount, null, default(CancellationToken), true)
.ProcessAcl());
}
/// <summary>
/// Api to call Acl Processor. Runs Acl Processor and returns the results.
/// </summary>
/// <param name="path">Root path from where the Acl recursive processor will start</param>
/// <param name="client">ADLS Client</param>
/// <param name="aclEntries">Acl Entries to change</param>
/// <param name="type">Type of Acl Job: Acl modify or Acl set or acl remove</param>
/// <param name="threadCount">Custom number of threads</param>
/// <param name="aclStatus">Status of progress</param>
/// <param name="cancelToken">Cancellationtoken</param>
/// <returns></returns>
internal static AclProcessorStats RunAclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount = -1, IProgress <AclProcessorStats> aclStatus = null, CancellationToken cancelToken = default(CancellationToken))
{
return(new AclProcessor(path, client, aclEntries, type, threadCount, aclStatus, cancelToken).ProcessAcl());
}
/// <summary>
/// Changes Acl recursively
/// </summary>
/// <param name="path">Input path</param>
/// <param name="accountName">Account name</param>
/// <param name="aclToSet">List of acl to set</param>
/// <param name="aclChangeType">Type of al change- Modify, Set, Remove</param>
/// <param name="concurrency">Concurrency- number of parallel operations</param>
/// <param name="aclCmdlet">Cmdlet for acl change. This is only for printing progress. If passed null, then no progress tracking is done</param>
/// <param name="trackProgress"></param>
/// <param name="cmdletCancellationToken">Cancellationtoken for cmdlet</param>
public AclProcessorStats ChangeAclRecursively(string path, string accountName, List <AclEntry> aclToSet,
RequestedAclType aclChangeType, int concurrency, Cmdlet aclCmdlet, bool trackProgress, CancellationToken cmdletCancellationToken)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
// Currently mockadlsclient signature is different, once that gets fixed, we can remove this
if (client.GetType() != typeof(MockAdlsClient))
{
System.Progress <AclProcessorStats> progressTracker = null;
ProgressRecord progress = null;
// If passing null, then we do not want progreess tracking
if (trackProgress)
{
progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000),
string.Format($"Recursive acl change for path {path}"),
$"Type of Acl Change: {aclChangeType}")
{
PercentComplete = 0
};
// On update from the Data Lake store uploader, capture the progress.
progressTracker = new System.Progress <AclProcessorStats>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = 0;
progress.Activity =
$"Files enumerated: {e.FilesProcessed} Directories enumerated:{e.DirectoryProcessed}";
}
};
}
AclProcessorStats status = null;
Task aclTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
status = client.ChangeAcl(path, aclToSet, aclChangeType, concurrency, progressTracker,
cmdletCancellationToken);
}, cmdletCancellationToken);
if (trackProgress || _isDebugEnabled)
{
TrackTaskProgress(aclTask, aclCmdlet, progress, cmdletCancellationToken);
if (trackProgress && !cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, aclCmdlet);
}
}
else
{
WaitForTask(aclTask, cmdletCancellationToken);
}
return(status);
}
return(client.ChangeAcl(path, aclToSet, aclChangeType, concurrency));
}
/// <summary>
/// Api to call Acl Processor. Runs Acl Processor and returns the results.
/// </summary>
/// <param name="path">Root path from where the Acl recursive processor will start</param>
/// <param name="client">ADLS Client</param>
/// <param name="aclEntries">Acl Entries to change</param>
/// <param name="type">Type of Acl Job: Acl modify or Acl set or acl remove</param>
/// <param name="threadCount">Custom number of threads</param>
/// <returns></returns>
internal static AclProcessorStats RunAclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount = -1)
{
return(new AclProcessor(path, client, aclEntries, type, threadCount).ProcessAcl());
}
/// <summary>
/// Internal test Api to verify Acl Processor. Runs Acl verifier and returns number of files and directories processed correctly.
/// </summary>
/// <param name="path">Root path from where the Acl recursive verifier will start</param>
/// <param name="client">ADLS Client</param>
/// <param name="aclEntries">Acl Entries to verify</param>
/// <param name="type">Type of Acl Job: Acl modify verify or Acl set verify or acl remove verify</param>
/// <param name="threadCount">Custom number of threads</param>
/// <param name="verifyFile">Verification file</param>
/// <param name="ignoreError">If passed true, then we will ignore the error and dump the error in verifyFile. Pass this true only if verifyFile is not null</param>
/// <param name="statusTracker">Status Tracker</param>
/// <param name="cancelToken">Cancel Token</param>
/// <returns></returns>
internal static AclProcessorStats RunAclVerifier(string path, AdlsClient client, List <AclEntry> aclEntries,
RequestedAclType type, int threadCount = -1, string verifyFile = null, bool ignoreError = false, IProgress <AclProcessorStats> statusTracker = null, CancellationToken cancelToken = default(CancellationToken))
{
return(new AclProcessor(path, client, aclEntries, type, threadCount, statusTracker, cancelToken, true, verifyFile, ignoreError)
.ProcessAcl());
}
/// <summary>
///
/// </summary>
/// <param name="path"></param>
/// <param name="accountName"></param>
/// <param name="aclToSet"></param>
/// <param name="aclChangeType"></param>
/// <param name="concurrency"></param>
public AclProcessorStats ChangeAclRecursively(string path, string accountName, List <AclEntry> aclToSet,
RequestedAclType aclChangeType, int concurrency = -1)
{
return(AdlsClientFactory.GetAdlsClient(accountName, _context).ChangeAcl(path, aclToSet, aclChangeType, concurrency));
}
/// <summary>
/// Currently the recursive entities need to be created separately for mock testing
/// </summary>
/// <param name="path"></param>
/// <param name="aclEntries"></param>
/// <param name="type"></param>
/// <param name="threadCount"></param>
/// <returns></returns>
public override AclProcessorStats ChangeAcl(string path, List <AclEntry> aclEntries, RequestedAclType type, int threadCount = -1)
{
int numDirs = 0, numFiles = 0;
foreach (var directoryEntriesKey in _directoryEntries.Keys)
{
if (directoryEntriesKey.StartsWith(path))
{
switch (type)
{
case RequestedAclType.ModifyAcl: ModifyAclEntries(_directoryEntries[directoryEntriesKey].Entry.FullName, aclEntries);
break;
case RequestedAclType.RemoveAcl:
RemoveAclEntries(_directoryEntries[directoryEntriesKey].Entry.FullName, aclEntries);
break;
case RequestedAclType.SetAcl:
SetAcl(_directoryEntries[directoryEntriesKey].Entry.FullName, aclEntries);
break;
}
if (_directoryEntries[directoryEntriesKey].Entry.Type == DirectoryEntryType.DIRECTORY)
{
numDirs++;
}
else
{
numFiles++;
}
}
}
return(new AclProcessorStats(numFiles, numDirs));
}
/// <summary>
/// Currently the recursive entities need to be created separately for mock testing
/// </summary>
/// <param name="path"></param>
/// <param name="aclEntries"></param>
/// <param name="type"></param>
/// <param name="threadCount"></param>
/// <param name="statusTracker"></param>
/// <param name="cancelToken"></param>
/// <returns></returns>
public override AclProcessorStats ChangeAcl(string path, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, IProgress <AclProcessorStats> statusTracker, CancellationToken cancelToken)
{
return(ChangeAcl(path, aclEntries, type, threadCount));
}