/// <summary>
/// Delegate method run by stats thread
/// </summary>
private void VerifyFileDumpRun()
{
while (true)
{
var entry = _incorrectFileList.Poll();
if (entry == null)
{
break;
}
_incorrectVerifyFileStream.WriteLine(entry);
}
_incorrectVerifyFileStream.Dispose();
}
private void RunProcess()
{
while (true)
{
var job = _queue.Poll();
if (job == null)
{
_queue.Add(null);
return;
}
_runMethod(job);
}
}
/// <summary>
/// The run method of each thread worker. It polls for a directory from the queue. Then calls listStatus for that directory.
/// If it gets any sub-directory it adds it to the queue so that it can be processed again later.For each file/sub-directory it updates the file/directory/size variables
/// </summary>
private void Run()
{
while (true)
{
DirectoryEntry der = _queue.Poll();
//GetException should be put here because some threads might be in waiting state and come back and see exception
if (GetException() != null || der == null) //der==null: Time to finish as all other threads have no entries
{
_queue.Add(null); //Poison block to notify other threads to close
return;
}
if (CancelToken.IsCancellationRequested)//Check if operation is cancelled
{
AdlsException excep = new AdlsException("Content summary processing cancelled")
{
Ex = new OperationCanceledException()
};
SetException(excep);
_queue.Add(null);
return;
}
try
{
foreach (var dir in Client.EnumerateDirectory(der.FullName))
{
if (dir.Type == DirectoryEntryType.DIRECTORY)
{
Interlocked.Increment(ref _directoryCount);
if (!(dir.Attribute != null && dir.Attribute.Any(attr => attr == DirectoryEntryAttributeType.Link)))
{
_queue.Add(dir);
}
}
else
{
Interlocked.Increment(ref _fileCount);
Interlocked.Add(ref _totalBytes, dir.Length);
}
}
}
catch (AdlsException ex)
{
if (ex.HttpStatus != HttpStatusCode.NotFound) //Do not stop summary if the file is deleted
{
SetException(ex); //Sets the global exception to signal other threads to close
_queue.Add(null); //Handle corner cases like when exception is raised other threads can be in wait state
return;
}
}
}
}
private void RunMetaDataWrite()
{
while (true)
{
var entry = _recordQueue.Poll();
if (entry == null)
{
break;
}
if (entry.Equals(FlushIndicator))
{
_writeStream.Flush();
}
else
{
_writeStream.WriteLine(entry);
}
}
_writeStream.Dispose();
}