Example #1
0
        /// <summary>
        /// Start a crawl from a root folder, authenticating as the process user
        /// </summary>
        /// <param name="ds"></param>
        /// <param name="rootpath"></param>
        /// <param name="driver"></param>
        public void CrawlRoot(DataStore ds, string rootpath)
        {
            ConsoleWriter.WriteInfo("Crawling " + rootpath);

            //get the existing folders for comparison
            try
            {
                TransactionResult <Dictionary <string, Folder> > existfolderstx = Reader.GetAllFoldersAsDict(rootpath, this.Driver);
                this._existingfolders = existfolderstx.Result;
                ConsoleWriter.WriteInfo("Found " + this._existingfolders.Count + " folders in database in " + existfolderstx.ElapsedMilliseconds.TotalMilliseconds + "ms");
            }
            catch (Exception e)
            {
                ConsoleWriter.WriteError("Error reading existing folders from database " + rootpath + ": " + e.Message);
                return;
            }

            //create the datastore node
            try
            {
                _timer.Start();
                Writer.SendDatastore(ds, this.Driver);
                Writer.AttachRootToDataStore(ds, rootpath.ToLower(), this.Driver);
            }
            catch (Exception e)
            {
                ConsoleWriter.WriteError("Error adding datastore " + ds.Name + ": " + e.Message);
                return;
            }

            //start at root and recurse down
            try
            {
                CrawlerThreadWrapper threadwrapper = new CrawlerThreadWrapper(this, 0);
                threadwrapper.IsRoot     = true;
                threadwrapper.Path       = rootpath;
                threadwrapper.PermParent = null;

                //start a new thread for the crawl
                int threadnum = ThreadCounter.RequestThread();
                threadwrapper.ThreadNumber = threadnum;
                threadwrapper.IsNewThread  = true;
                ThreadPool.QueueUserWorkItem(threadwrapper.Crawl);

                //wait for threads to finish
                while (true)
                {
                    Thread.Sleep(5000);
                    if (ThreadCounter.ActiveThreadCount == 0)
                    {
                        break;
                    }
                }

                Writer.FlushFolderQueue(this.Driver);
                _timer.Stop();
                ConsoleWriter.ClearProgress();
                ConsoleWriter.WriteInfo("Crawled file system " + rootpath + " in " + _timer.Elapsed);
                ConsoleWriter.WriteLine();
            }
            catch (Exception e)
            {
                Writer.FlushFolderQueue(this.Driver);
                _timer.Stop();
                ConsoleWriter.WriteError("Error crawling file system " + rootpath + ": " + e.Message);
                ConsoleWriter.WriteLine();
                return;
            }

            //cleanup folders that have changed
            try
            {
                _timer.Restart();
                ConsoleWriter.WriteInfo("Cleaning up ");
                Writer.CleanupChangedFolders(rootpath, this.Driver);
                Writer.CleanupConnections(rootpath, this.Driver);
                _timer.Stop();
                ConsoleWriter.WriteInfo("Clean finished in " + _timer.Elapsed);
                ConsoleWriter.WriteLine();
            }
            catch (Exception e)
            {
                _timer.Stop();
                ConsoleWriter.ClearProgress();
                ConsoleWriter.WriteError("Error cleaning up folders " + rootpath + ": " + e.Message);
                ConsoleWriter.WriteLine();
            }

            ConsoleWriter.WriteInfo("Found " + Writer.FolderCount + " folders with permissions applied");
        }
Example #2
0
        /// <summary>
        /// Crawl subdirectories recursively. permparent is the last parent folder that had new permissions set
        /// </summary>
        /// <param name="path"></param>
        /// <param name="permparent"></param>
        public void Crawl()
        {
            bool   connected     = false;
            string newpermparent = this.PermParent;

            try
            {
                Folder f = this.QueryFolder(new DirectoryInfo(this.Path), this.PermParent, this.IsRoot);
                if ((f.PermissionCount > 0) || this.IsRoot)
                {
                    newpermparent = this.Path;
                    this._parent.Writer.UpdateFolder(f, this._parent.Driver);
                }

                connected = true;
            }
            catch (Exception e)
            {
                ConsoleWriter.WriteError("Error connecting to " + this.Path + ": " + e.Message);
                Folder f = new Folder()
                {
                    Blocked             = true,
                    Path                = this.Path,
                    Name                = this.Path,
                    PermParent          = this.PermParent,
                    InheritanceDisabled = true,
                    Depth               = this.Depth
                };
                this._parent.Writer.UpdateFolder(f, this._parent.Driver);
            }

            if (connected == true)
            {
                try
                {
                    foreach (string subdirpath in Directory.EnumerateDirectories(this.Path))
                    {
                        CrawlerThreadWrapper subwrapper = new CrawlerThreadWrapper(this._parent, this.Depth + 1);
                        subwrapper.Path       = subdirpath;
                        subwrapper.PermParent = newpermparent;
                        subwrapper.IsRoot     = false;

                        int threadnum = ThreadCounter.RequestThread();

                        if (threadnum != -1)
                        {
                            subwrapper.ThreadNumber = threadnum;
                            subwrapper.IsNewThread  = true;
                            ThreadPool.QueueUserWorkItem(subwrapper.Crawl);
                        }
                        else
                        {
                            subwrapper.ThreadNumber = this.ThreadNumber;
                            subwrapper.Crawl();
                        }
                    }
                }
                catch (Exception e)
                {
                    ConsoleWriter.WriteError("Error encountered while enumerating directories in " + this.Path + ": " + e.Message);
                }
            }

            if (this.IsNewThread == true)
            {
                ThreadCounter.Release(this.ThreadNumber);
                ConsoleWriter.WriteProgress(this.ThreadNumber + " | " + "Released thread", this.ThreadNumber);
            }
        }