/// <summary>
/// Refreshes the state of the object.
/// </summary>
/// <remarks>
/// FileSystemInfo.Refresh takes a snapshot of the file from the current file system. Refresh cannot correct the underlying file system even if the file system returns incorrect or outdated information. This can happen on platforms such as Windows 98.
/// Calls must be made to Refresh before attempting to get the attribute information, or the information will be outdated.
/// </remarks>
public override void Refresh()
{
mFileSystemEntryInfo = File.GetFileSystemEntryInfo(FullPath);
if (mFileSystemEntryInfo != null && mFileSystemEntryInfo.IsFile)
{
mExists = true;
}
else
{
mExists = false;
mFileSystemEntryInfo = null;
}
}
/// <summary>
/// Refreshes the state of the object.
/// </summary>
/// <remarks>
/// FileSystemInfo.Refresh takes a snapshot of the file from the current file system. Refresh cannot correct the underlying file system even if the file system returns incorrect or outdated information. This can happen on platforms such as Windows 98.
/// Calls must be made to Refresh before attempting to get the attribute information, or the information will be outdated.
/// </remarks>
public override void Refresh()
{
try
{
mFileSystemEntryInfo = File.GetFileSystemEntryInfo(FullPath);
if (mFileSystemEntryInfo != null && mFileSystemEntryInfo.IsDirectory)
{
mExists = true;
}
}
catch (Exception)
{
mExists = false;
mFileSystemEntryInfo = null;
}
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("Kruacm8p1B6RFAw2WMnKcEqkQcnWRkF3RmPSOzlW");
SetupNLog();
SetupPatterns();
_logger = LogManager.GetCurrentClassLogger();
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to search. Either this or -d is required");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to recursively process. Either this or -f is required");
_fluentCommandLineParser.Setup(arg => arg.SaveTo)
.As('o')
.WithDescription("File to save results to");
_fluentCommandLineParser.Setup(arg => arg.GetAscii)
.As('a')
.SetDefault(true)
.WithDescription("If set, look for ASCII strings. Default is true. Use -a false to disable");
_fluentCommandLineParser.Setup(arg => arg.GetUnicode)
.As('u')
.SetDefault(true)
.WithDescription("If set, look for Unicode strings. Default is true. Use -u false to disable");
_fluentCommandLineParser.Setup(arg => arg.MinimumLength)
.As('m').SetDefault(3).WithDescription("Minimum string length. Default is 3");
_fluentCommandLineParser.Setup(arg => arg.BlockSizeMb)
.As('b').SetDefault(512).WithDescription("Chunk size in MB. Valid range is 1 to 1024. Default is 512");
_fluentCommandLineParser.Setup(arg => arg.Quiet)
.As('q').SetDefault(false).WithDescription("Quiet mode (Do not show header or total number of hits)");
_fluentCommandLineParser.Setup(arg => arg.QuietQuiet)
.As('s')
.SetDefault(false)
.WithDescription(
"Really Quiet mode (Do not display hits to console. Speeds up processing when using -o)");
_fluentCommandLineParser.Setup(arg => arg.MaximumLength)
.As('x').SetDefault(-1).WithDescription("Maximum string length. Default is unlimited\r\n");
_fluentCommandLineParser.Setup(arg => arg.GetPatterns)
.As('p').SetDefault(false).WithDescription("Display list of built in regular expressions");
_fluentCommandLineParser.Setup(arg => arg.LookForString)
.As("ls")
.SetDefault(string.Empty)
.WithDescription("String to look for. When set, only matching strings are returned");
_fluentCommandLineParser.Setup(arg => arg.LookForRegex)
.As("lr")
.SetDefault(string.Empty)
.WithDescription("Regex to look for. When set, only strings matching the regex are returned");
_fluentCommandLineParser.Setup(arg => arg.StringFile)
.As("fs")
.SetDefault(string.Empty)
.WithDescription("File containing strings to look for. When set, only matching strings are returned");
_fluentCommandLineParser.Setup(arg => arg.RegexFile)
.As("fr")
.SetDefault(string.Empty)
.WithDescription(
"File containing regex patterns to look for. When set, only strings matching regex patterns are returned\r\n");
_fluentCommandLineParser.Setup(arg => arg.AsciiRange)
.As("ar")
.SetDefault("[\x20-\x7E]")
.WithDescription(
@"Range of characters to search for in 'Code page' strings. Specify as a range of characters in hex format and enclose in quotes. Default is [\x20 -\x7E]");
_fluentCommandLineParser.Setup(arg => arg.UnicodeRange)
.As("ur")
.SetDefault("[\u0020-\u007E]")
.WithDescription(
"Range of characters to search for in Unicode strings. Specify as a range of characters in hex format and enclose in quotes. Default is [\\u0020-\\u007E]\r\n");
_fluentCommandLineParser.Setup(arg => arg.CodePage)
.As("cp")
.SetDefault(1252)
.WithDescription(
"Code page to use. Default is 1252. Use the Identifier value for code pages at https://goo.gl/ig6DxW");
_fluentCommandLineParser.Setup(arg => arg.FileMask)
.As("mask")
.SetDefault(string.Empty)
.WithDescription(
"When using -d, file mask to search for. * and ? are supported. This option has no effect when using -f");
_fluentCommandLineParser.Setup(arg => arg.RegexOnly)
.As("ro")
.SetDefault(false)
.WithDescription(
"When true, list the string matched by regex pattern vs string the pattern was found in (This may result in duplicate strings in output. ~ denotes approx. offset)");
_fluentCommandLineParser.Setup(arg => arg.ShowOffset)
.As("off")
.SetDefault(false)
.WithDescription(
$"Show offset to hit after string, followed by the encoding (A={_fluentCommandLineParser.Object.CodePage}, U=Unicode)\r\n");
_fluentCommandLineParser.Setup(arg => arg.SortAlpha)
.As("sa").SetDefault(false).WithDescription("Sort results alphabetically");
_fluentCommandLineParser.Setup(arg => arg.SortLength)
.As("sl").SetDefault(false).WithDescription("Sort results by length");
var header =
$"bstrings version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/bstrings";
var footer = @"Examples: bstrings.exe -f ""C:\Temp\UsrClass 1.dat"" --ls URL" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someFile.txt"" --lr guid" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\aBigFile.bin"" --fs c:\temp\searchStrings.txt --fr c:\temp\searchRegex.txt -s" +
"\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --mask ""*.dll""" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --ar ""[\x20-\x37]""" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --cp 10007" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --ls test" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someOtherFile.txt"" --lr cc -sa" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someOtherFile.txt"" --lr cc -sa -m 15 -x 22" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\UsrClass 1.dat"" --ls mui -sl" + "\r\n\t ";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (_fluentCommandLineParser.Object.GetPatterns)
{
_logger.Warn("Name \t\tDescription");
foreach (var regExPattern in RegExPatterns.OrderBy(t => t.Key))
{
var desc = RegExDesc[regExPattern.Key];
_logger.Info($"{regExPattern.Key}\t{desc}");
}
_logger.Info("");
_logger.Info("To use a built in pattern, supply the Name to the --lr switch\r\n");
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("Either -f or -d is required. Exiting");
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false &&
!File.Exists(_fluentCommandLineParser.Object.File) &&
_fluentCommandLineParser.Object.FileMask.Length == 0)
{
_logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting");
return;
}
if (_fluentCommandLineParser.Object.Directory.IsNullOrEmpty() == false &&
!Directory.Exists(_fluentCommandLineParser.Object.Directory) &&
_fluentCommandLineParser.Object.FileMask.Length == 0)
{
_logger.Warn($"Directory '{_fluentCommandLineParser.Object.Directory}' not found. Exiting");
return;
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(header);
_logger.Info("");
}
var files = new List <string>();
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
files.Add(Path.GetFullPath(_fluentCommandLineParser.Object.File));
}
else
{
try
{
if (_fluentCommandLineParser.Object.FileMask.Length > 0)
{
files.AddRange(Directory.EnumerateFiles(Path.GetFullPath(_fluentCommandLineParser.Object.Directory),
_fluentCommandLineParser.Object.FileMask, SearchOption.AllDirectories));
}
else
{
files.AddRange(Directory.EnumerateFiles(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), "*",
SearchOption.AllDirectories));
}
}
catch (Exception ex)
{
_logger.Error(
$"Error getting files in '{_fluentCommandLineParser.Object.Directory}'. Error message: {ex.Message}");
return;
}
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Command line: {string.Join(" ", args)}");
_logger.Info("");
}
StreamWriter sw = null;
var globalCounter = 0;
var globalHits = 0;
double globalTimespan = 0;
var withBoundaryHits = false;
if (_fluentCommandLineParser.Object.SaveTo.IsNullOrEmpty() == false && _fluentCommandLineParser.Object.SaveTo.Length > 0)
{
_fluentCommandLineParser.Object.SaveTo = _fluentCommandLineParser.Object.SaveTo.TrimEnd('\\');
var dir = Path.GetDirectoryName(_fluentCommandLineParser.Object.SaveTo);
if (dir != null && Directory.Exists(dir) == false)
{
try
{
Directory.CreateDirectory(dir);
}
catch (Exception)
{
_logger.Warn(
$"Invalid path: '{_fluentCommandLineParser.Object.SaveTo}'. Results will not be saved to a file.");
_logger.Info("");
_fluentCommandLineParser.Object.SaveTo = string.Empty;
}
}
else
{
if (dir == null)
{
_logger.Warn($"Invalid path: '{_fluentCommandLineParser.Object.SaveTo}");
_fluentCommandLineParser.Object.SaveTo = string.Empty;
}
}
if (_fluentCommandLineParser.Object.SaveTo.Length > 0 && !_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Saving hits to '{_fluentCommandLineParser.Object.SaveTo}'");
_logger.Info("");
}
if (_fluentCommandLineParser.Object.SaveTo.Length > 0)
{
// try
// {
// File.Create(_fluentCommandLineParser.Object.SaveTo);
// }
// catch (Exception e)
// {
// _logger.Fatal($"Unable to create output file '{_fluentCommandLineParser.Object.SaveTo}'! Check permissions and try again! Error: {e.Message}");
// return;
// }
sw = new StreamWriter(_fluentCommandLineParser.Object.SaveTo, true);
}
}
foreach (var file in files)
{
if (File.Exists(file) == false)
{
_logger.Warn($"'{file}' does not exist! Skipping");
}
_sw = new Stopwatch();
_sw.Start();
var counter = 0;
var hits = new HashSet <string>();
var regPattern = _fluentCommandLineParser.Object.LookForRegex;
if (RegExPatterns.ContainsKey(_fluentCommandLineParser.Object.LookForRegex))
{
regPattern = RegExPatterns[_fluentCommandLineParser.Object.LookForRegex];
}
if (regPattern.Length > 0 && !_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Searching via RegEx pattern: {regPattern}");
_logger.Info("");
}
var minLength = 3;
if (_fluentCommandLineParser.Object.MinimumLength > 0)
{
minLength = _fluentCommandLineParser.Object.MinimumLength;
}
var maxLength = -1;
if (_fluentCommandLineParser.Object.MaximumLength > minLength)
{
maxLength = _fluentCommandLineParser.Object.MaximumLength;
}
var chunkSizeMb = _fluentCommandLineParser.Object.BlockSizeMb < 1 ||
_fluentCommandLineParser.Object.BlockSizeMb > 1024
? 512
: _fluentCommandLineParser.Object.BlockSizeMb;
var chunkSizeBytes = chunkSizeMb * 1024 * 1024;
var fileSizeBytes = new FileInfo(file).Length;
var bytesRemaining = fileSizeBytes;
long offset = 0;
var chunkIndex = 1;
var totalChunks = fileSizeBytes / chunkSizeBytes + 1;
var hsuffix = totalChunks == 1 ? "" : "s";
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
$"Searching {totalChunks:N0} chunk{hsuffix} ({chunkSizeMb} MB each) across {GetSizeReadable(fileSizeBytes)} in '{file}'");
_logger.Info("");
}
try
{
MappedStream ms = null;
try
{
var fs =
File.Open(File.GetFileSystemEntryInfo(file).LongFullPath, FileMode.Open, FileAccess.Read,
FileShare.Read, PathFormat.LongFullPath);
ms = MappedStream.FromStream(fs, Ownership.None);
}
catch (Exception)
{
_logger.Warn($"Unable to open file directly. This usually means the file is in use. Switching to raw access\r\n");
}
if (ms == null)
{
//raw mode
var ss = OpenFile(file);
ms = MappedStream.FromStream(ss, Ownership.None);
}
using (ms)
{
while (bytesRemaining > 0)
{
if (bytesRemaining <= chunkSizeBytes)
{
chunkSizeBytes = (int)bytesRemaining;
}
var chunk = new byte[chunkSizeBytes];
ms.Read(chunk, 0, chunkSizeBytes);
if (_fluentCommandLineParser.Object.GetUnicode)
{
var uh = GetUnicodeHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in uh)
{
hits.Add(h);
}
}
if (_fluentCommandLineParser.Object.GetAscii)
{
var ah = GetAsciiHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in ah)
{
hits.Add(h);
}
}
offset += chunkSizeBytes;
bytesRemaining -= chunkSizeBytes;
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
$"Chunk {chunkIndex:N0} of {totalChunks:N0} finished. Total strings so far: {hits.Count:N0} Elapsed time: {_sw.Elapsed.TotalSeconds:N3} seconds. Average strings/sec: {hits.Count/_sw.Elapsed.TotalSeconds:N0}");
}
chunkIndex += 1;
}
//do chunk boundary checks to make sure we get everything and not split things
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
"Primary search complete. Looking for strings across chunk boundaries...");
}
bytesRemaining = fileSizeBytes;
chunkSizeBytes = chunkSizeMb * 1024 * 1024;
offset = chunkSizeBytes - _fluentCommandLineParser.Object.MinimumLength * 10 * 2;
//move starting point backwards for our starting point
chunkIndex = 0;
var boundaryChunkSize = _fluentCommandLineParser.Object.MinimumLength * 10 * 2 * 2;
//grab the same # of bytes on both sides of the boundary
while (bytesRemaining > 0)
{
if (offset + boundaryChunkSize > fileSizeBytes)
{
break;
}
var chunk = new byte[boundaryChunkSize];
ms.Read(chunk, 0, boundaryChunkSize);
if (_fluentCommandLineParser.Object.GetUnicode)
{
var uh = GetUnicodeHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in uh)
{
hits.Add(" " + h);
}
if (withBoundaryHits == false && uh.Count > 0)
{
withBoundaryHits = uh.Count > 0;
}
}
if (_fluentCommandLineParser.Object.GetAscii)
{
var ah = GetAsciiHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in ah)
{
hits.Add(" " + h);
}
if (withBoundaryHits == false && ah.Count > 0)
{
withBoundaryHits = true;
}
}
offset += chunkSizeBytes;
bytesRemaining -= chunkSizeBytes;
chunkIndex += 1;
}
}
}
catch (Exception ex)
{
_logger.Info("");
_logger.Error($"Error: {ex.Message}");
}
_sw.Stop();
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("Search complete.");
_logger.Info("");
}
if (_fluentCommandLineParser.Object.SortAlpha)
{
_logger.Info("Sorting alphabetically...");
_logger.Info("");
var tempList = hits.ToList();
tempList.Sort();
hits = new HashSet <string>(tempList);
}
else if (_fluentCommandLineParser.Object.SortLength)
{
_logger.Info("Sorting by length...");
_logger.Info("");
var tempList = SortByLength(hits.ToList()).ToList();
hits = new HashSet <string>(tempList);
}
var fileStrings = new HashSet <string>();
var regexStrings = new HashSet <string>();
//set up highlighting
if (_fluentCommandLineParser.Object.LookForString.Length > 0)
{
fileStrings.Add(_fluentCommandLineParser.Object.LookForString);
}
if (_fluentCommandLineParser.Object.LookForRegex.Length > 0)
{
regexStrings.Add(regPattern);
}
if (_fluentCommandLineParser.Object.StringFile.IsNullOrEmpty() == false || _fluentCommandLineParser.Object.RegexFile.IsNullOrEmpty() == false)
{
if (_fluentCommandLineParser.Object.StringFile.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.StringFile))
{
fileStrings.UnionWith(new HashSet <string>(File.ReadAllLines(_fluentCommandLineParser.Object.StringFile)));
}
else
{
_logger.Error($"Strings file '{_fluentCommandLineParser.Object.StringFile}' not found.");
}
}
if (_fluentCommandLineParser.Object.RegexFile.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.RegexFile))
{
regexStrings.UnionWith(new HashSet <string>(File.ReadAllLines(_fluentCommandLineParser.Object.RegexFile)));
}
else
{
_logger.Error($"Regex file '{_fluentCommandLineParser.Object.RegexFile}' not found.");
}
}
}
AddHighlightingRules(fileStrings.ToList());
if (_fluentCommandLineParser.Object.RegexOnly == false)
{
AddHighlightingRules(regexStrings.ToList(), true);
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("Processing strings...");
_logger.Info("");
}
foreach (var hit in hits)
{
if (hit.Length == 0)
{
continue;
}
if (fileStrings.Count > 0 || regexStrings.Count > 0)
{
foreach (var fileString in fileStrings)
{
if (fileString.Trim().Length == 0)
{
continue;
}
if (hit.IndexOf(fileString, StringComparison.InvariantCultureIgnoreCase) < 0)
{
continue;
}
counter += 1;
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
var hitoffset = "";
if (_fluentCommandLineParser.Object.ShowOffset)
{
hitoffset = $"~{hit.Split('\t').Last()}";
}
foreach (var regString in regexStrings)
{
if (regString.Trim().Length == 0)
{
continue;
}
try
{
var reg1 = new Regex(regString,
RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
if (reg1.IsMatch(hit) == false)
{
continue;
}
counter += 1;
if (_fluentCommandLineParser.Object.RegexOnly)
{
foreach (var match in reg1.Matches(hit))
{
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info($"{match}\t{hitoffset}");
}
sw?.WriteLine($"{match}\t{hitoffset}");
}
}
else
{
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
}
catch (Exception ex)
{
_logger.Error($"Error setting up regular expression '{regString}': {ex.Message}");
}
}
}
else
{
//dump all strings
counter += 1;
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
}
if (_fluentCommandLineParser.Object.Quiet)
{
continue;
}
var suffix = counter == 1 ? "" : "s";
_logger.Info("");
if (withBoundaryHits)
{
_logger.Info("** Strings prefixed with 2 spaces are hits found across chunk boundaries **");
_logger.Info("");
}
_logger.Info(
$"Found {counter:N0} string{suffix} in {_sw.Elapsed.TotalSeconds:N3} seconds. Average strings/sec: {hits.Count/_sw.Elapsed.TotalSeconds:N0}");
globalCounter += counter;
globalHits += hits.Count;
globalTimespan += _sw.Elapsed.TotalSeconds;
if (files.Count > 1)
{
_logger.Info(
"-------------------------------------------------------------------------------------\r\n");
}
}
if (sw != null)
{
sw.Flush();
sw.Close();
}
if (!_fluentCommandLineParser.Object.Quiet && files.Count > 1)
{
var suffix = globalCounter == 1 ? "" : "s";
_logger.Info("");
_logger.Info(
$"Found {globalCounter:N0} string{suffix} in {globalTimespan:N3} seconds. Average strings/sec: {globalHits/globalTimespan:N0}");
}
}
/// <summary>
/// The daemon code calls this occasionally to poke it into action to do work
/// </summary>
/// <param name="daemon"></param>
public void ExecuteBackgroundProcess(Daemon daemon)
{
// We don't want to start watching files until the library is loaded...
if (!(LibraryRef?.Xlibrary.LibraryIsLoaded ?? false))
{
Logging.Info("Library is not yet loaded, so waiting before watching...");
// Indicate that the library may still not have been changed...
FolderContentsHaveChanged = true;
return;
}
// Update our folder system watcher if necessary
CheckIfFolderNameHasChanged();
// If the current folder is blank, do nothing
if (String.IsNullOrEmpty(configured_folder_to_watch))
{
return;
}
// If the folder does not exist, do nothing
if (!Directory.Exists(configured_folder_to_watch))
{
Logging.Info("Watched folder {0} does not exist: watching this directory has been disabled.", configured_folder_to_watch);
return;
}
// If the folder or its contents has not changed since the last time, do nothing
if (!FolderContentsHaveChanged)
{
return;
}
if (!ConfigurationManager.IsEnabled(nameof(FolderWatcher)))
{
Logging.Info("Watched folder {0} will not be watched/scanned due to Developer Override setting {1}=false", configured_folder_to_watch, nameof(FolderWatcher));
return;
}
Stopwatch breathing_time = Stopwatch.StartNew();
Logging.Debug("FolderWatcher BEGIN");
// To recover from a fatal library failure and re-indexing attempt for very large libraries,
// we're better off processing a limited number of source files as we'll be able to see
// *some* results more quickly and we'll have a working, though yet incomplete,
// index in *reasonable time*.
//
// To reconstruct the entire index will take a *long* time. We grow the index and other meta
// stores a bunch-of-files at a time and then repeat the entire maintenance process until
// we'll be sure to have run out of files to process for sure...
// Mark that we are now processing the folder
while (TestAndReset_FolderContentsHaveChanged())
{
// If this library is busy, skip it for now
if (Library.IsBusyAddingPDFs || Library.IsBusyRegeneratingTags)
{
Logging.Debug特("FolderWatcher: Not daemon processing any library that is busy with adds...");
FolderContentsHaveChanged = true;
break;
}
if (ShutdownableManager.Instance.IsShuttingDown)
{
Logging.Debug特("FolderWatcher: Breaking out of outer processing loop due to daemon termination");
FolderContentsHaveChanged = true;
break;
}
if (Qiqqa.Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
{
Logging.Debug特("FolderWatcher: Breaking out of outer processing loop due to DisableAllBackgroundTasks");
FolderContentsHaveChanged = true;
break;
}
if (LibraryRef == null || folder_watcher_manager?.TypedTarget == null)
{
Logging.Debug特("FolderWatcher: Breaking out of outer processing loop due to disposed library and/or watch manager");
FolderContentsHaveChanged = true;
break;
}
if (!ConfigurationManager.IsEnabled(nameof(FolderWatcher)))
{
Logging.Info("Watched folder {0} will not be watched/scanned due to Developer Override setting {1}=false", configured_folder_to_watch, nameof(FolderWatcher));
break;
}
// reset counters for logging/reporting:
watch_stats.Reset(daemon);
// If we get this far then there might be some work to do in the folder...
Stopwatch clk = Stopwatch.StartNew();
//
// Summary:
// [AlphaFS] Specifies a set of custom filters to be used with enumeration methods
// of Alphaleonis.Win32.Filesystem.Directory, e.g., Alphaleonis.Win32.Filesystem.Directory.EnumerateDirectories(System.String),
// Alphaleonis.Win32.Filesystem.Directory.EnumerateFiles(System.String), or Alphaleonis.Win32.Filesystem.Directory.EnumerateFileSystemEntries(System.String).
//
// Remarks:
// Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters allows scenarios in
// which files/directories being enumerated by the methods of Alphaleonis.Win32.Filesystem.Directory
// class are accepted only if they match the search pattern, attributes (see Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.SkipReparsePoints),
// and optionally also the custom criteria tested in the method whose delegate is
// specified in Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.InclusionFilter.
// These criteria could be, e.g., file size exceeding some threshold, pathname matches
// a complex regular expression, etc. If the enumeration process is set to be recursive
// (see Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.Recursive) and
// Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.RecursionFilter is specified,
// the directory is traversed recursively only if it matches the custom criteria
// in Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.RecursionFilter method.
// This allows, for example, custom handling of junctions and symbolic links, e.g.,
// detection of cycles. If any error occurs during the enumeration and the enumeration
// process is not set to ignore errors (see Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.ContinueOnException),
// an exception is thrown unless the error is handled (filtered out) by the method
// specified in Alphaleonis.Win32.Filesystem.DirectoryEnumerationFilters.ErrorFilter
// (if specified). The method may, for example, consume the error by reporting it
// in a log, so that the enumeration continues as in the case of Alphaleonis.Win32.Filesystem.DirectoryEnumerationOptions.ContinueOnException
// option but the user will be informed about errors.
//
global_watch_stats.Inc();
DirectoryEnumerationFilters filter = new DirectoryEnumerationFilters();
filter.ErrorFilter = DecideIfErrorDuringDirScan;
filter.InclusionFilter = DecideIfIncludeDuringDirScan;
filter.RecursionFilter = DecideIfRecurseDuringDirScan;
// Note: don't use the CancellationToken, just throw an exception in the InclusionFilter when it's time to abort the scan.
//filter.CancellationToken = null;
IEnumerable <string> filenames_in_folder = Directory.EnumerateFiles(configured_folder_to_watch,
DirectoryEnumerationOptions.Files |
DirectoryEnumerationOptions.BasicSearch |
//DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.LargeCache |
DirectoryEnumerationOptions.Recursive,
filter);
// SearchOption.AllDirectories);
Logging.Debug特("Directory.EnumerateFiles took {0} ms", clk.ElapsedMilliseconds);
// Do NOT count files which are already present in our library/DB,
// despite the fact that those also *do* take time and effort to check
// in the code above.
//
// The issue here is that when we would import files A,B,C,D,E,F,G,H,I,J,K,
// we would do so in tiny batches, resulting in a rescan after each batch
// where the already processed files will be included in the set, but must
// be filtered out as 'already in there' in the code above.
// Iff we had counted *all* files we inspect from the Watch Directory,
// we would never make it batch the first batch as then our count limit
// would trigger already for every round through here!
List <string> filenames_that_are_new = new List <string>();
foreach (string filename in filenames_in_folder)
{
Logging.Info("FolderWatcher: {0} of {1} files have been processed/inspected (total {2} scanned, {3} skipped, {4} ignored)", watch_stats.processed_file_count, watch_stats.processing_file_count, watch_stats.scanned_file_count, watch_stats.skipped_file_count, watch_stats.scanned_file_count - watch_stats.skipped_file_count - watch_stats.processing_file_count);
try
{
// check the file once again: it MAY have disappeared while we were slowly scanning the remainder of the dirtree.
FileSystemEntryInfo info = File.GetFileSystemEntryInfo(filename);
watch_stats.processing_file_count++;
Logging.Info("FolderWatcher is importing {0}", filename);
filenames_that_are_new.Add(filename);
}
catch (Exception ex)
{
Logging.Error(ex, "Folder Watcher: skipping file {0} due to file I/O error {1}", filename, ex.Message);
}
}
Logging.Debug特("Directory.EnumerateFiles took {0} ms", clk.ElapsedMilliseconds);
// Create the import records
List <FilenameWithMetadataImport> filename_with_metadata_imports = new List <FilenameWithMetadataImport>();
foreach (var filename in filenames_that_are_new)
{
filename_with_metadata_imports.Add(new FilenameWithMetadataImport
{
filename = filename,
tags = new HashSet <string>(tags)
});
#if false
// delay until the PDF has actually been processed completely!
//
// Add this file to the list of processed files...
folder_watcher_manager.RememberProcessedFile(filename);
#endif
}
// Get the library to import all these new files
if (filename_with_metadata_imports.Count > 0)
{
ImportingIntoLibrary.AddNewPDFDocumentsToLibraryWithMetadata_SYNCHRONOUS(LibraryRef, true, filename_with_metadata_imports.ToArray());
// TODO: refactor the ImportingIntoLibrary class
}
watch_stats.processed_file_count = watch_stats.processing_file_count;
Logging.Info("FolderWatcher: {0} of {1} files have been processed/inspected (total {2} scanned, {3} skipped, {4} ignored)", watch_stats.processed_file_count, watch_stats.processing_file_count, watch_stats.scanned_file_count, watch_stats.skipped_file_count, watch_stats.scanned_file_count - watch_stats.skipped_file_count - watch_stats.processing_file_count);
if (watch_stats.index_processing_clock.ElapsedMilliseconds >= FolderWatcher.MAX_SECONDS_PER_ITERATION)
{
Logging.Info("FolderWatcher: Taking a nap due to MAX_SECONDS_PER_ITERATION: {0} seconds consumed, {1} threads pending", watch_stats.index_processing_clock.ElapsedMilliseconds / 1E3, SafeThreadPool.QueuedThreadCount);
watch_stats.daemon.Sleep(SECONDS_TO_RELAX_PER_ITERATION);
watch_stats.index_processing_clock.Restart();
}
Logging.Debug("FolderWatcher End-Of-Round ({0} ms)", clk.ElapsedMilliseconds);
}
Logging.Debug("FolderWatcher END");
}
