private PropertyManager(AdlsClient client, bool getAclProperty, bool getDiskUsage, string saveFileName, bool saveToLocal, int numThreads, bool displayFiles, bool displayConsistentAcl, long maxDepth, CancellationToken cancelToken = default(CancellationToken))
{
Client = client;
SaveToLocal = saveToLocal;
if (string.IsNullOrWhiteSpace(saveFileName))
{
throw new ArgumentNullException(nameof(saveFileName));
}
DumpFileName = saveFileName;
GetSizeProperty = getDiskUsage;
GetAclProperty = getAclProperty;
_numThreads = numThreads <= 0 ? AdlsClient.DefaultNumThreads : numThreads;
DisplayFiles = displayFiles;
HideConsistentAclTree = GetAclProperty && displayConsistentAcl;
MaxDepth = maxDepth;
ConsumerQueue = new PriorityQueueWrapper <BaseJob>(_numThreads);
PropertyWriterQueue = new PriorityQueueWrapper <BaseJob>();
_cancelToken = cancelToken;
Stream underLyingStream;
if (saveToLocal)
{
Utils.CreateParentDirectory(DumpFileName);
underLyingStream = new FileStream(DumpFileName, FileMode.Create, FileAccess.ReadWrite);
}
else
{
underLyingStream = client.CreateFile(DumpFileName, IfExists.Overwrite);
}
PropertyDumpWriter = new StreamWriter(underLyingStream);
WriteHeader();
}
private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, IProgress <AclProcessorStats> aclStatusTracker, CancellationToken cancelToken, bool verify = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount <= 0 ? AdlsClient.DefaultNumThreads: threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
if (aclEntries == null || aclEntries.Count == 0)
{
throw new ArgumentException("Input acl is null or empty");
}
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
if (FileAclEntries.Count == 0 && AclLog.IsDebugEnabled)
{
AclLog.Debug("AclEntries for file are empty so input acl must be containing default acls");
}
Type = type;
_isVerify = verify;
_aclStatusTracker = aclStatusTracker;
_cancelToken = cancelToken;
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":",AclEntries)}{(_isVerify?", RunInVerifyMode":string.Empty)}");
}
}
private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, IProgress <AclProcessorStats> aclStatusTracker, CancellationToken cancelToken, bool verify = false, string verifyFile = null, bool ignoreVerifyTimeErrors = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount <= 0 ? AdlsClient.DefaultNumThreads : threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
if (aclEntries == null || aclEntries.Count == 0)
{
throw new ArgumentException("Input acl is null or empty");
}
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
if (FileAclEntries.Count == 0 && AclLog.IsDebugEnabled)
{
AclLog.Debug("AclEntries for file are empty so input acl must be containing default acls");
}
Type = type;
_isVerify = verify;
_aclStatusTracker = aclStatusTracker;
_cancelToken = cancelToken;
// If verify file is passed we have to setup a thread and a filestream to write to the file
if (verify && !string.IsNullOrEmpty(verifyFile))
{
_ignoreVerifyTimeErrors = ignoreVerifyTimeErrors;
_incorrectVerifyFile = verifyFile;
_incorrectFileList = new QueueWrapper <string>(-1);
Utils.CreateParentDirectory(_incorrectVerifyFile);
_incorrectVerifyFileStream = new StreamWriter(new FileStream(_incorrectVerifyFile, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
AutoFlush = true
};
}
_linkPaths = new ConcurrentBag <string>();
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":", AclEntries)}{(_isVerify ? ", RunInVerifyMode" : string.Empty)}");
}
}
protected FileTransferCommon(string srcPath, string destPath, AdlsClient client, int numThreads, IfExists doOverwrite, IProgress <TransferStatus> progressTracker, bool notRecurse, bool resume, bool ingressOrEgressTest, long chunkSize, string metaDataPath, CancellationToken cancelToken, string metaDataInfo = null)
{
if (string.IsNullOrWhiteSpace(srcPath))
{
throw new ArgumentException(nameof(srcPath));
}
SourcePath = srcPath;
DestPath = destPath;
Client = client;
// If ingress or egress test then simply overwrite destination
DoOverwrite = ingressOrEgressTest ? IfExists.Overwrite : doOverwrite;
ProgressTracker = progressTracker;
IngressOrEgressTest = ingressOrEgressTest;
NumConsumerThreads = numThreads <= 0 ? AdlsClient.DefaultNumThreads : numThreads;
ChunkSize = chunkSize;
NotRecurse = notRecurse;
metaDataInfo = string.IsNullOrEmpty(metaDataInfo) ? ChunkSize.ToString() : metaDataInfo + $",ChunkSize:{chunkSize},{(NotRecurse ? "NotRecurse" : "Recurse")}";
RecordedMetadata = new TransferLog(resume, metaDataPath, metaDataInfo);
Status = new TransferStatus();
ConsumerQueue = new PriorityQueueWrapper <BaseJob>();
CancelToken = cancelToken;
}
private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, bool verify = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount < 0 ? AdlsClient.DefaultNumThreads: threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
Type = type;
_isVerify = verify;
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":",AclEntries)}{(_isVerify?", RunInVerifyMode":string.Empty)}");
}
}
public CrossPartitionRangePageAsyncEnumerator(
IFeedRangeProvider feedRangeProvider,
CreatePartitionRangePageAsyncEnumerator <TPage, TState> createPartitionRangeEnumerator,
IComparer <PartitionRangePageAsyncEnumerator <TPage, TState> > comparer,
int?maxConcurrency,
CancellationToken cancellationToken,
CrossFeedRangeState <TState> state = default)
{
this.feedRangeProvider = feedRangeProvider ?? throw new ArgumentNullException(nameof(feedRangeProvider));
this.createPartitionRangeEnumerator = createPartitionRangeEnumerator ?? throw new ArgumentNullException(nameof(createPartitionRangeEnumerator));
this.cancellationToken = cancellationToken;
this.lazyEnumerators = new AsyncLazy <IQueue <PartitionRangePageAsyncEnumerator <TPage, TState> > >(async(ITrace trace, CancellationToken token) =>
{
ReadOnlyMemory <FeedRangeState <TState> > rangeAndStates;
if (state != default)
{
rangeAndStates = state.Value;
}
else
{
// Fan out to all partitions with default state
List <FeedRangeEpk> ranges = await feedRangeProvider.GetFeedRangesAsync(trace, token);
List <FeedRangeState <TState> > rangesAndStatesBuilder = new List <FeedRangeState <TState> >(ranges.Count);
foreach (FeedRangeInternal range in ranges)
{
rangesAndStatesBuilder.Add(new FeedRangeState <TState>(range, default));
}
rangeAndStates = rangesAndStatesBuilder.ToArray();
}
List <BufferedPartitionRangePageAsyncEnumerator <TPage, TState> > bufferedEnumerators = new List <BufferedPartitionRangePageAsyncEnumerator <TPage, TState> >(rangeAndStates.Length);
for (int i = 0; i < rangeAndStates.Length; i++)
{
FeedRangeState <TState> feedRangeState = rangeAndStates.Span[i];
PartitionRangePageAsyncEnumerator <TPage, TState> enumerator = createPartitionRangeEnumerator(feedRangeState);
BufferedPartitionRangePageAsyncEnumerator <TPage, TState> bufferedEnumerator = new BufferedPartitionRangePageAsyncEnumerator <TPage, TState>(enumerator, cancellationToken);
bufferedEnumerators.Add(bufferedEnumerator);
}
if (maxConcurrency.HasValue)
{
await ParallelPrefetch.PrefetchInParallelAsync(bufferedEnumerators, maxConcurrency.Value, trace, token);
}
IQueue <PartitionRangePageAsyncEnumerator <TPage, TState> > queue;
if (comparer == null)
{
queue = new QueueWrapper <PartitionRangePageAsyncEnumerator <TPage, TState> >(
new Queue <PartitionRangePageAsyncEnumerator <TPage, TState> >(bufferedEnumerators));
}
else
{
queue = new PriorityQueueWrapper <PartitionRangePageAsyncEnumerator <TPage, TState> >(
new PriorityQueue <PartitionRangePageAsyncEnumerator <TPage, TState> >(
bufferedEnumerators,
comparer));
}
return(queue);
});
}
public CrossPartitionRangePageAsyncEnumerator(
IFeedRangeProvider feedRangeProvider,
CreatePartitionRangePageAsyncEnumerator <TPage, TState> createPartitionRangeEnumerator,
IComparer <PartitionRangePageAsyncEnumerator <TPage, TState> > comparer,
int?maxConcurrency,
CancellationToken cancellationToken,
CrossPartitionState <TState> state = default)
{
this.feedRangeProvider = feedRangeProvider ?? throw new ArgumentNullException(nameof(feedRangeProvider));
this.createPartitionRangeEnumerator = createPartitionRangeEnumerator ?? throw new ArgumentNullException(nameof(createPartitionRangeEnumerator));
this.cancellationToken = cancellationToken;
this.lazyEnumerators = new AsyncLazy <IQueue <PartitionRangePageAsyncEnumerator <TPage, TState> > >(async(CancellationToken token) =>
{
IReadOnlyList <(FeedRangeInternal, TState)> rangeAndStates;
if (state != default)
{
rangeAndStates = state.Value;
}
else
{
// Fan out to all partitions with default state
IEnumerable <FeedRangeInternal> ranges = await feedRangeProvider.GetFeedRangesAsync(token);
List <(FeedRangeInternal, TState)> rangesAndStatesBuilder = new List <(FeedRangeInternal, TState)>();
foreach (FeedRangeInternal range in ranges)
{
rangesAndStatesBuilder.Add((range, default));
}
rangeAndStates = rangesAndStatesBuilder;
}
List <BufferedPartitionRangePageAsyncEnumerator <TPage, TState> > bufferedEnumerators = rangeAndStates
.Select(rangeAndState =>
{
PartitionRangePageAsyncEnumerator <TPage, TState> enumerator = createPartitionRangeEnumerator(rangeAndState.Item1, rangeAndState.Item2);
BufferedPartitionRangePageAsyncEnumerator <TPage, TState> bufferedEnumerator = new BufferedPartitionRangePageAsyncEnumerator <TPage, TState>(enumerator, cancellationToken);
return(bufferedEnumerator);
})
.ToList();
if (maxConcurrency.HasValue)
{
await ParallelPrefetch.PrefetchInParallelAsync(bufferedEnumerators, maxConcurrency.Value, token);
}
IQueue <PartitionRangePageAsyncEnumerator <TPage, TState> > queue;
if (comparer == null)
{
queue = new QueueWrapper <PartitionRangePageAsyncEnumerator <TPage, TState> >(
new Queue <PartitionRangePageAsyncEnumerator <TPage, TState> >(bufferedEnumerators));
}
else
{
queue = new PriorityQueueWrapper <PartitionRangePageAsyncEnumerator <TPage, TState> >(
new PriorityQueue <PartitionRangePageAsyncEnumerator <TPage, TState> >(
bufferedEnumerators,
comparer));
}
return(queue);
});
}
