private async Task <bool> TraverseTree(string root, int maxTasks)
{
string currentDir = root;
using (var throttler = new SemaphoreSlim(maxTasks))
{
var postTaskTasks = new List <PFTask>();
while (currentDir != null)
{
if (currentDir == root)
{
// First time get the directories syncroniously to populate the DB with directories to run on
IEnumerable <string> directories = Directory.EnumerateDirectories(currentDir);
List <string> lst = new List <string>();
foreach (string dir in directories)
{
try
{
DirectoryInfo di = new DirectoryInfo(dir);
lst.Add(di.FullName);
Log.Trace("Added folder: " + di.FullName + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
}
catch (Exception ex)
{
Log.Info(ex, "User doesn't have permissions to folder: " + dir);
}
}
//await SMBDal.BulkAddFolders(directories.ToList());
await SMBDal.BulkAddFolders(lst);
}
else
{
Log.Trace("Before wait: " + currentDir + ", wait: " + throttler.CurrentCount + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
await throttler.WaitAsync();
Log.Trace("After wait: " + currentDir + ", wait: " + throttler.CurrentCount + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
string value = currentDir.ToString();
PFTask t = new PFTask();
t.Task = Task.Run(() => ProcessDir(value), t.CancellationToken).ContinueWith(tsk => release(throttler));
postTaskTasks.Add(t);
Log.Trace("After add task: " + currentDir + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
}
// Fetch the next directory to traverse and do it again
currentDir = await SMBIterator.GetNextFolderForTraverse();
Log.Trace("GetNextFolderForTraverse returned: " + currentDir + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
// Clean the completed tasks from the array
foreach (PFTask t in postTaskTasks)
{
if (t.Task.IsCompleted || t.Task.IsCanceled || t.Task.IsFaulted)
{
_tasksToRemove.Add(t);
}
}
foreach (PFTask t in _tasksToRemove)
{
postTaskTasks.Remove(t);
}
_tasksToRemove = new List <PFTask>();
if (currentDir == null)
{
Log.Trace("WaitAll" + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
Task.WaitAll(postTaskTasks.Select(t => t.Task).ToArray());
currentDir = await SMBIterator.GetNextFolderForTraverse();
Log.Trace("Got directory after wait all: " + currentDir + ", Thread Id: " + Thread.CurrentThread.ManagedThreadId);
}
}
}
return(true);
}
public async Task <ScanResult> ScanNext(Repository repo)
{
IteratorItem item = await SMBIterator.NextItemAsync();
return(await ScanNext(repo, item));
}
public async Task <bool> Scan(Repository repo)
{
bool scanDone = false;
var postTaskTasks = new List <PFTask>();
object SpinLock = new object();
PrepCredentials(repo);
using (var throttler = new SemaphoreSlim(_maxFilesScanCocur))
{
// Scan files
while (!scanDone)
{
bool semValue = await throttler.WaitAsync(_taskTimeout);
if (!semValue)
{
// See what tasks are hanging for more than 30 seconds and kill them
foreach (PFTask t in postTaskTasks)
{
if (t.StartTime.Subtract(DateTime.UtcNow).Seconds > _taskTimeout)
{
t.TokenSource.Cancel();
}
}
}
IteratorItem item = null;
lock (SpinLock)
{
// Get next item for iteration
item = SMBIterator.NextItem();
}
if (item != null)
{
// process the next item
PFTask tsk = new PFTask();
tsk.Task = Task.Run <ScanResult>(() => ScanNext(repo, item), tsk.CancellationToken).ContinueWith(t => release(throttler));
postTaskTasks.Add(tsk);
}
else
{
scanDone = true;
}
// Clean the completed tasks from the wait array
foreach (PFTask t in postTaskTasks)
{
if (t.Task.IsCompleted || t.Task.IsCanceled || t.Task.IsFaulted)
{
_tasksToRemove.Add(t);
}
}
foreach (PFTask t in _tasksToRemove)
{
postTaskTasks.Remove(t);
t.Task.Dispose();
}
_tasksToRemove = new List <PFTask>();
if (scanDone)
{
// Wait for all the tasks to finish before exiting
Task.WaitAll(postTaskTasks.Select(i => i.Task).ToArray(), 30000);
}
}
}
Counter.PrintAll();
return(true);
}
