static void Main(string[] args)
{
//Console.SetBufferSize(Console.BufferWidth, 10000);
//Console.WriteLine(Console.BufferHeight);
int i = 1;
int dotsmult = 1;
int dotscount = 0;
ConsoleWriter.SetProgressLineCount(2);
while (i < 12000)
{
dotscount = dotscount + dotsmult;
string dots = new String('.', dotscount);
ConsoleWriter.WriteInfo(i.ToString() + ":" + Console.BufferHeight + ":" + dots);
ConsoleWriter.WriteProgress(i.ToString() + ":1", 1);
ConsoleWriter.WriteProgress(i.ToString() + ":2", 2);
if (i % 300 == 0)
{
dotsmult = dotsmult * -1;
}
i++;
//Console.WriteLine(i);
}
//Console.WriteLine(Console.BufferHeight);
ConsoleWriter.ShowProgress = false;
ConsoleWriter.ClearProgress();
ConsoleWriter.WriteInfo("Tester finished");
ConsoleWriter.WriteLine("1");
ConsoleWriter.WriteLine("1");
ConsoleWriter.WriteLine("1");
ConsoleWriter.WriteLine();
//ConsoleWriter.ClearProgress();
ConsoleWriter.WriteLine();
ConsoleWriter.Write("Exiting.");
for (int j = 0; j < 3; j++)
{
System.Threading.Thread.Sleep(500);
ConsoleWriter.Write(".");
}
Console.ReadLine();
}
static void Main(string[] args)
{
Console.Title = "BirdsNest File System Scanner";
Stopwatch steptimer = new Stopwatch();
Stopwatch totaltimer = new Stopwatch();
Dictionary <string, NetworkCredential> credentials = new Dictionary <string, NetworkCredential>();
List <DataStore> datastores = new List <DataStore>();
string _appdir = AppDomain.CurrentDomain.BaseDirectory;
string neoconfigfile = _appdir + @"\config\neoconfig.json";
string configfile = _appdir + @"\config\fsconfig.json";
//int relcounter = 0;
bool batchmode = false;
string scanid = ShortGuid.NewGuid().ToString();
IDriver driver = null;
ConsoleWriter.InitLine(1);
totaltimer.Start();
try
{
foreach (string arg in args)
{
string[] param = arg.Split(new[] { ":" }, 2, StringSplitOptions.None);
switch (param[0].ToUpper())
{
case "/?":
ShowUsage();
Environment.Exit(0);
break;
case "/CONFIG":
configfile = param[1];
break;
case "/BATCH":
batchmode = true;
break;
default:
break;
}
}
}
catch
{
Console.WriteLine("There is a problem with arguments: " + string.Join(" ", args));
Console.WriteLine("");
ShowUsage();
Environment.Exit(ErrorCodes.ArgumentsError);
}
try
{
using (Configuration config = Configuration.LoadConfiguration(configfile))
{
try { config.Validate(); }
catch (ArgumentException e)
{
ConsoleWriter.WriteError(e.Message);
if (batchmode == false)
{
Console.ReadLine();
}
Environment.Exit(ErrorCodes.ConfigValidationError);
}
foreach (Credential cred in config.Credentials)
{
NetworkCredential netcred = new NetworkCredential(cred.Username, cred.Password, cred.Domain);
credentials.Add(cred.ID, netcred);
}
datastores = config.Datastores;
ConsoleWriter.ShowProgress = config.ShowProgress;
ConsoleWriter.SetProgressLineCount(config.MaxThreads);
}
}
catch (Exception e)
{
ConsoleWriter.WriteError("There was an error loading your configuration: " + e.Message);
if (batchmode == false)
{
Console.ReadLine();
}
Environment.Exit(ErrorCodes.ConfigLoadError);
}
try
{
using (NeoConfiguration config = NeoConfiguration.LoadConfigurationFile(neoconfigfile))
{
driver = Neo4jConnector.ConnectToNeo(config);
}
}
catch (Exception e)
{
ConsoleWriter.WriteError("There was an error loading your neo4j configuration: " + e.Message);
if (batchmode == false)
{
Console.ReadLine();
}
Environment.Exit(ErrorCodes.NeoConfigLoadError);
}
ConsoleWriter.WriteInfo("Initialising file system scanner, scan ID: " + scanid);
foreach (DataStore ds in datastores)
{
foreach (FileSystem fs in ds.FileSystems)
{
if (string.IsNullOrEmpty(fs.Path))
{
ConsoleWriter.WriteWarning("Filesystem missing \"path\" property");
continue;
}
if (string.IsNullOrEmpty(fs.ID))
{
ConsoleWriter.WriteError("Filesystem does have have an ID configured: " + fs.Path);
ConsoleWriter.WriteError("A random ID has been generated for you to use in your config: " + ShortGuid.NewGuid().ToString());
if (batchmode == false)
{
ConsoleWriter.WriteLine("Press enter to continue");
Console.ReadLine();
}
continue;
}
Crawler crawler = new Crawler(driver, fs, scanid);
NetworkCredential fscred;
if (!string.IsNullOrEmpty(fs.CredentialID) && (credentials.TryGetValue(fs.CredentialID, out fscred)))
{
crawler.CrawlRoot(ds, fs.Path, fscred);
}
else
{
crawler.CrawlRoot(ds, fs.Path);
}
}
}
totaltimer.Stop();
ConsoleWriter.ShowProgress = false;
ConsoleWriter.ClearProgress();
TimeSpan totaltime = new TimeSpan(totaltimer.ElapsedTicks);
ConsoleWriter.WriteLine("Finished in " + totaltime.ToString());
ConsoleWriter.WriteLine("Done");
if (batchmode == true)
{
ConsoleWriter.Write("Exiting.");
for (int i = 0; i < 3; i++)
{
System.Threading.Thread.Sleep(500);
ConsoleWriter.Write(".");
}
}
else
{
ConsoleWriter.WriteLine();
ConsoleWriter.WriteLine("Press enter to exit");
Console.ReadLine();
}
}
/// <summary>
/// Start a crawl from a root folder, authenticating as the process user
/// </summary>
/// <param name="ds"></param>
/// <param name="rootpath"></param>
/// <param name="driver"></param>
public void CrawlRoot(DataStore ds, string rootpath)
{
ConsoleWriter.WriteInfo("Crawling " + rootpath);
//get the existing folders for comparison
try
{
TransactionResult <Dictionary <string, Folder> > existfolderstx = Reader.GetAllFoldersAsDict(rootpath, this.Driver);
this._existingfolders = existfolderstx.Result;
ConsoleWriter.WriteInfo("Found " + this._existingfolders.Count + " folders in database in " + existfolderstx.ElapsedMilliseconds.TotalMilliseconds + "ms");
}
catch (Exception e)
{
ConsoleWriter.WriteError("Error reading existing folders from database " + rootpath + ": " + e.Message);
return;
}
//create the datastore node
try
{
_timer.Start();
Writer.SendDatastore(ds, this.Driver);
Writer.AttachRootToDataStore(ds, rootpath.ToLower(), this.Driver);
}
catch (Exception e)
{
ConsoleWriter.WriteError("Error adding datastore " + ds.Name + ": " + e.Message);
return;
}
//start at root and recurse down
try
{
CrawlerThreadWrapper threadwrapper = new CrawlerThreadWrapper(this, 0);
threadwrapper.IsRoot = true;
threadwrapper.Path = rootpath;
threadwrapper.PermParent = null;
//start a new thread for the crawl
int threadnum = ThreadCounter.RequestThread();
threadwrapper.ThreadNumber = threadnum;
threadwrapper.IsNewThread = true;
ThreadPool.QueueUserWorkItem(threadwrapper.Crawl);
//wait for threads to finish
while (true)
{
Thread.Sleep(5000);
if (ThreadCounter.ActiveThreadCount == 0)
{
break;
}
}
Writer.FlushFolderQueue(this.Driver);
_timer.Stop();
ConsoleWriter.ClearProgress();
ConsoleWriter.WriteInfo("Crawled file system " + rootpath + " in " + _timer.Elapsed);
ConsoleWriter.WriteLine();
}
catch (Exception e)
{
Writer.FlushFolderQueue(this.Driver);
_timer.Stop();
ConsoleWriter.WriteError("Error crawling file system " + rootpath + ": " + e.Message);
ConsoleWriter.WriteLine();
return;
}
//cleanup folders that have changed
try
{
_timer.Restart();
ConsoleWriter.WriteInfo("Cleaning up ");
Writer.CleanupChangedFolders(rootpath, this.Driver);
Writer.CleanupConnections(rootpath, this.Driver);
_timer.Stop();
ConsoleWriter.WriteInfo("Clean finished in " + _timer.Elapsed);
ConsoleWriter.WriteLine();
}
catch (Exception e)
{
_timer.Stop();
ConsoleWriter.ClearProgress();
ConsoleWriter.WriteError("Error cleaning up folders " + rootpath + ": " + e.Message);
ConsoleWriter.WriteLine();
}
ConsoleWriter.WriteInfo("Found " + Writer.FolderCount + " folders with permissions applied");
}
