private Task <string[]> ParseFiles(string[] files)
{
return(new Task <string[]>(() =>
{
List <string> validFiles = new List <string>(files);
Mutex mtx = new Mutex();
if (FilterExtensions.Length > 0)
{
Parallel.ForEach(validFiles.ToArray(), fName =>
{
if (!FileExtensionHandler.EndsWithExtension(fName, FilterExtensions))
{
mtx.WaitOne();
validFiles.Remove(fName);
mtx.ReleaseMutex();
}
});
}
if (BlockExtensions.Length > 0)
{
Parallel.ForEach(validFiles.ToArray(), fName =>
{
if (FileExtensionHandler.EndsWithExtension(fName, BlockExtensions))
{
mtx.WaitOne();
validFiles.Remove(fName);
mtx.ReleaseMutex();
}
});
}
if (!string.IsNullOrEmpty(Year))
{
Parallel.ForEach(validFiles.ToArray(), fName =>
{
FileInfo fInfo = new FileInfo(fName);
string lastWrite = File.GetLastWriteTime(fInfo.FullName).Date.ToString();
if (!lastWrite.Contains(Year))
{
mtx.WaitOne();
validFiles.Remove(fName);
mtx.ReleaseMutex();
}
});
}
if (SearchTerms.Length > 0)
{
Parallel.ForEach(validFiles.ToArray(), fName =>
{
if (FileExtensionHandler.HasCleanExtension(fName) &&
!FileContainsStrings(fName))
{
mtx.WaitOne();
validFiles.Remove(fName);
mtx.ReleaseMutex();
}
else if (!FileExtensionHandler.HasCleanExtension(fName))
{
Console.WriteLine($"[-] Removing file {fName} as it cannot be parsed for search terms.");
mtx.WaitOne();
validFiles.Remove(fName);
mtx.ReleaseMutex();
}
});
}
return validFiles.ToArray();
}));
}
static void ParseDirectory(string path,
string pattern = null,
string[] ext_whitelist = null,
string[] ext_blacklist = null,
string[] searchterms = null,
string year = null)
{
string[] files = null;
string[] directories = null;
// Get all the directories in the passed path
directories = GetDirectoriesInDirectory(path);
if (directories.Length > 0)
{
// For each directory, do the same file parsing and add to the thread queue.
//Console.WriteLine("There's {0} directoreis to comb through", directories.Length);
foreach (string dir in directories)
{
//Console.WriteLine("Searching {0}", dir);
Thread t = new Thread(() => ParseDirectory(dir, pattern, ext_whitelist, ext_blacklist, searchterms, year));
t.Start();
//Console.WriteLine("Started thread");
_pool.WaitOne(500);
//Console.WriteLine("Waited");
runningThreads.Add(t);
_pool.Release(1);
//Console.WriteLine("released");
}
}
files = GetFilesInDirectory(path, pattern);
List <string> validExtensionFiles = new List <string>();
if (ext_whitelist != null)
{
foreach (string fName in files)
{
//Console.WriteLine("Checking {0}", fName);
if (FileExtensionHandler.EndsWithExtension(fName, ext_whitelist))
{
//Console.WriteLine("{0} looks valid", fName);
validExtensionFiles.Add(fName);
}
}
}
else if (ext_blacklist != null)
{
foreach (string fName in files)
{
if (!FileExtensionHandler.EndsWithExtension(fName, ext_blacklist))
{
//Console.WriteLine("{0} looks valid");
validExtensionFiles.Add(fName);
}
}
}
else
{
foreach (string f in files)
{
validExtensionFiles.Add(f);
}
}
if (year != null && year != "")
{
foreach (string fname in validExtensionFiles.ToArray())
{
FileInfo fileInfo = new FileInfo(fname);
string lastWrite = File.GetLastWriteTime(fileInfo.FullName).Date.ToString();
if (!lastWrite.Contains(year))
{
validExtensionFiles.Remove(fname);
}
}
}
if (searchterms != null)
{
foreach (string fName in validExtensionFiles)
{
if (FileExtensionHandler.HasCleanExtension(fName) && FileContainsStrings(fName, searchterms))
{
//Console.WriteLine("Waiting for pool thread");
_pool.WaitOne(500);
//Console.WriteLine("added {0}", fName);
matchingFiles.Add(fName);
_pool.Release(1);
}
}
}
else
{
_pool.WaitOne(500);
foreach (string fName in validExtensionFiles)
{
//Console.WriteLine("yolo {0}", fName);
matchingFiles.Add(fName);
}
_pool.Release(1);
}
//Console.WriteLine("Done searching {0}", path);
}