/// <summary>
/// Start a crawl from a root folder, authenticating as the process user
/// </summary>
/// <param name="ds"></param>
/// <param name="rootpath"></param>
/// <param name="driver"></param>
public void CrawlRoot(DataStore ds, string rootpath)
{
ConsoleWriter.WriteInfo("Crawling " + rootpath);
//get the existing folders for comparison
try
{
TransactionResult <Dictionary <string, Folder> > existfolderstx = Reader.GetAllFoldersAsDict(rootpath, this.Driver);
this._existingfolders = existfolderstx.Result;
ConsoleWriter.WriteInfo("Found " + this._existingfolders.Count + " folders in database in " + existfolderstx.ElapsedMilliseconds.TotalMilliseconds + "ms");
}
catch (Exception e)
{
ConsoleWriter.WriteError("Error reading existing folders from database " + rootpath + ": " + e.Message);
return;
}
//create the datastore node
try
{
_timer.Start();
Writer.SendDatastore(ds, this.Driver);
Writer.AttachRootToDataStore(ds, rootpath.ToLower(), this.Driver);
}
catch (Exception e)
{
ConsoleWriter.WriteError("Error adding datastore " + ds.Name + ": " + e.Message);
return;
}
//start at root and recurse down
try
{
CrawlerThreadWrapper threadwrapper = new CrawlerThreadWrapper(this, 0);
threadwrapper.IsRoot = true;
threadwrapper.Path = rootpath;
threadwrapper.PermParent = null;
//start a new thread for the crawl
int threadnum = ThreadCounter.RequestThread();
threadwrapper.ThreadNumber = threadnum;
threadwrapper.IsNewThread = true;
ThreadPool.QueueUserWorkItem(threadwrapper.Crawl);
//wait for threads to finish
while (true)
{
Thread.Sleep(5000);
if (ThreadCounter.ActiveThreadCount == 0)
{
break;
}
}
Writer.FlushFolderQueue(this.Driver);
_timer.Stop();
ConsoleWriter.ClearProgress();
ConsoleWriter.WriteInfo("Crawled file system " + rootpath + " in " + _timer.Elapsed);
ConsoleWriter.WriteLine();
}
catch (Exception e)
{
Writer.FlushFolderQueue(this.Driver);
_timer.Stop();
ConsoleWriter.WriteError("Error crawling file system " + rootpath + ": " + e.Message);
ConsoleWriter.WriteLine();
return;
}
//cleanup folders that have changed
try
{
_timer.Restart();
ConsoleWriter.WriteInfo("Cleaning up ");
Writer.CleanupChangedFolders(rootpath, this.Driver);
Writer.CleanupConnections(rootpath, this.Driver);
_timer.Stop();
ConsoleWriter.WriteInfo("Clean finished in " + _timer.Elapsed);
ConsoleWriter.WriteLine();
}
catch (Exception e)
{
_timer.Stop();
ConsoleWriter.ClearProgress();
ConsoleWriter.WriteError("Error cleaning up folders " + rootpath + ": " + e.Message);
ConsoleWriter.WriteLine();
}
ConsoleWriter.WriteInfo("Found " + Writer.FolderCount + " folders with permissions applied");
}
/// <summary>
/// Crawl subdirectories recursively. permparent is the last parent folder that had new permissions set
/// </summary>
/// <param name="path"></param>
/// <param name="permparent"></param>
public void Crawl()
{
bool connected = false;
string newpermparent = this.PermParent;
try
{
Folder f = this.QueryFolder(new DirectoryInfo(this.Path), this.PermParent, this.IsRoot);
if ((f.PermissionCount > 0) || this.IsRoot)
{
newpermparent = this.Path;
this._parent.Writer.UpdateFolder(f, this._parent.Driver);
}
connected = true;
}
catch (Exception e)
{
ConsoleWriter.WriteError("Error connecting to " + this.Path + ": " + e.Message);
Folder f = new Folder()
{
Blocked = true,
Path = this.Path,
Name = this.Path,
PermParent = this.PermParent,
InheritanceDisabled = true,
Depth = this.Depth
};
this._parent.Writer.UpdateFolder(f, this._parent.Driver);
}
if (connected == true)
{
try
{
foreach (string subdirpath in Directory.EnumerateDirectories(this.Path))
{
CrawlerThreadWrapper subwrapper = new CrawlerThreadWrapper(this._parent, this.Depth + 1);
subwrapper.Path = subdirpath;
subwrapper.PermParent = newpermparent;
subwrapper.IsRoot = false;
int threadnum = ThreadCounter.RequestThread();
if (threadnum != -1)
{
subwrapper.ThreadNumber = threadnum;
subwrapper.IsNewThread = true;
ThreadPool.QueueUserWorkItem(subwrapper.Crawl);
}
else
{
subwrapper.ThreadNumber = this.ThreadNumber;
subwrapper.Crawl();
}
}
}
catch (Exception e)
{
ConsoleWriter.WriteError("Error encountered while enumerating directories in " + this.Path + ": " + e.Message);
}
}
if (this.IsNewThread == true)
{
ThreadCounter.Release(this.ThreadNumber);
ConsoleWriter.WriteProgress(this.ThreadNumber + " | " + "Released thread", this.ThreadNumber);
}
}
