private void SetScreenSizeInPrefs()
{
var config = new IniParserConfiguration {
AllowDuplicateKeys = true, AllowDuplicateSections = true
};
foreach (var file in Directory.EnumerateFiles(Path.Combine(OutputFolder, "profiles"), "*refs.ini",
DirectoryEnumerationOptions.Recursive))
{
try
{
var parser = new FileIniDataParser(new IniDataParser(config));
var data = parser.ReadFile(file);
if (data.Sections["Display"] == null)
{
return;
}
if (data.Sections["Display"]["iSize W"] != null && data.Sections["Display"]["iSize H"] != null)
{
data.Sections["Display"]["iSize W"] = SystemParameters.ScreenWidth.ToString(CultureInfo.CurrentCulture);
data.Sections["Display"]["iSize H"] = SystemParameters.ScreenHeight.ToString(CultureInfo.CurrentCulture);
}
parser.WriteFile(file, data);
}
catch (Exception ex)
{
Utils.Log($"Skipping screen size remap for {file} due to parse error.");
continue;
}
}
}
private void CreateOutputMods()
{
Directory.EnumerateFiles(Path.Combine(OutputFolder, "profiles"), "settings.ini", DirectoryEnumerationOptions.Recursive).Do(f =>
{
var ini = f.LoadIniFile();
if (ini == null)
{
Utils.Log($"settings.ini is null for {f}, skipping");
return;
}
var overwrites = ini.custom_overwrites;
if (overwrites == null)
{
Utils.Log("No custom overwrites found, skipping");
return;
}
if (overwrites is SectionData data)
{
data.Coll.Do(keyData =>
{
var v = keyData.Value;
var mod = Path.Combine(OutputFolder, "mods", v);
if (!Directory.Exists(mod))
{
Directory.CreateDirectory(mod);
}
});
}
});
}
public IncludeZEditPatches(Compiler compiler) : base(compiler)
{
var zEditPath = FindzEditPath(compiler);
var havezEdit = zEditPath != null;
Utils.Log(havezEdit ? $"Found zEdit at {zEditPath}" : $"zEdit not detected, disabling zEdit routines");
if (!havezEdit)
{
_mergesIndexed = new Dictionary <string, zEditMerge>();
return;
}
var merges = Directory.EnumerateFiles(Path.Combine(zEditPath, "profiles"),
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive)
.Where(f => f.EndsWith("\\merges.json"))
.SelectMany(f => f.FromJSON <List <zEditMerge> >())
.GroupBy(f => (f.name, f.filename));
merges.Where(m => m.Count() > 1)
.Do(m =>
{
Utils.Warning(
$"WARNING, you have two patches named {m.Key.name}\\{m.Key.filename} in your zEdit profiles. We'll pick one at random, this probably isn't what you want.");
});
_mergesIndexed =
merges.ToDictionary(
m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename),
m => m.First());
}
private void IndexPath(string path)
{
var file_list = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories).ToList();
Utils.Log($"Updating the cache for {file_list.Count} files");
file_list.PMap(f => UpdateFile(f));
SyncToDisk();
}
private static IEnumerable <string> EnumerateAllFiles(string path, IList <string> patterns, bool includeArchive, bool recursive, bool followSymlinks)
{
// without filters, just enumerate files, which is faster
var fileOptions = baseFileOptions;
if (recursive)
{
fileOptions |= DirectoryEnumerationOptions.Recursive;
}
if (followSymlinks)
{
fileOptions &= ~DirectoryEnumerationOptions.SkipReparsePoints;
}
DirectoryEnumerationFilters fileFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
}
};
bool includeAllFiles = patterns.Count == 0 ||
(patterns.Count == 1 && (patterns[0] == "*.*" || patterns[0] == "*"));
if (!includeAllFiles)
{
fileFilters.InclusionFilter = fsei =>
{
foreach (string pattern in patterns)
{
if (WildcardMatch(fsei.FileName, pattern, true))
{
return(true);
}
}
if (includeArchive)
{
foreach (string pattern in ArchiveDirectory.Patterns)
{
if (WildcardMatch(fsei.FileName, pattern, true))
{
return(true);
}
}
}
return(false);
};
}
return(Directory.EnumerateFiles(path, fileOptions, fileFilters, PathFormat.FullPath));
}
private static List <string> EnumerateFiles(string path, ExportFormat format)
{
if (!path.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
path += Path.DirectorySeparatorChar;
}
return(Directory.EnumerateFiles(path, $"*.{format.ToString().ToLower()}", SearchOption.AllDirectories).ToList());
}
/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing Modlist directives");
var indexed = ModList.Directives.ToDictionary(d => d.To);
UpdateTracker.NextStep("Looking for files to delete");
await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
.PMap(Queue, UpdateTracker, f =>
{
var relative_to = f.RelativeTo(OutputFolder);
Utils.Status($"Checking if modlist file {relative_to}");
if (indexed.ContainsKey(relative_to) || f.IsInPath(DownloadFolder))
{
return;
}
Utils.Log($"Deleting {relative_to} it's not part of this modlist");
File.Delete(f);
});
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
Status($"Optimizing {d.To}");
var path = Path.Combine(OutputFolder, d.To);
if (!File.Exists(path))
{
return(null);
}
var fi = new FileInfo(path);
if (fi.Length != d.Size)
{
return(null);
}
return(path.FileHash() == d.Hash ? d : null);
}))
.Where(d => d != null)
.Do(d => indexed.Remove(d.To));
UpdateTracker.NextStep("Updating Modlist");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath[0])
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
public async Task HashArchives()
{
var hashResults = await Directory.EnumerateFiles(DownloadFolder)
.Where(e => !e.EndsWith(Consts.HashFileExtension))
.PMap(Queue, e => (e.FileHashCached(), e));
HashedArchives = hashResults
.OrderByDescending(e => File.GetLastWriteTime(e.Item2))
.GroupBy(e => e.Item1)
.Select(e => e.First())
.ToDictionary(e => e.Item1, e => e.Item2);
}
public static IErrorResponse CheckValidInstallPath(string path)
{
var ret = Utils.IsDirectoryPathValid(path);
if (!ret.Succeeded)
{
return(ret);
}
if (!Directory.Exists(path))
{
return(ErrorResponse.Success);
}
// Check folder does not have a wabbajack modlist
foreach (var file in Directory.EnumerateFiles(path, DirectoryEnumerationOptions.Recursive))
{
if (!File.Exists(file))
{
continue;
}
if (System.IO.Path.GetExtension(file).Equals(ExtensionManager.Extension))
{
return(ErrorResponse.Fail($"Cannot install into a folder with a wabbajack modlist inside of it."));
}
}
// Check folder is either empty, or a likely valid previous install
if (!Directory.IsEmpty(path))
{
// Some probably naive check, but should be a good starting point to improve later
if (!Directory.EnumerateFiles(path).Any(file =>
{
var fileName = Path.GetFileName(file);
if (fileName.Equals("ModOrganizer.exe", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
if (fileName.Equals("ModOrganizer.ini", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
return(false);
}))
{
return(ErrorResponse.Fail($"Cannot install into a non-empty folder that does not look like a previous WJ installation."));
}
}
return(ErrorResponse.Success);
}
private void UpdateArchive(VirtualFile f)
{
if (!f.IsStaged)
{
throw new InvalidDataException("Can't analyze an unstaged file");
}
var tmp_dir = Path.Combine(_stagedRoot, Guid.NewGuid().ToString());
Utils.Status($"Extracting Archive {Path.GetFileName(f.StagedPath)}");
FileExtractor.ExtractAll(f.StagedPath, tmp_dir);
Utils.Status($"Updating Archive {Path.GetFileName(f.StagedPath)}");
var entries = Directory.EnumerateFiles(tmp_dir, "*", SearchOption.AllDirectories)
.Select(path => path.RelativeTo(tmp_dir));
var new_files = entries.Select(e => {
var new_path = new string[f.Paths.Length + 1];
f.Paths.CopyTo(new_path, 0);
new_path[f.Paths.Length] = e;
var nf = new VirtualFile()
{
Paths = new_path,
};
nf._stagedPath = Path.Combine(tmp_dir, e);
Add(nf);
return(nf);
}).ToList();
// Analyze them
new_files.PMap(file =>
{
Utils.Status($"Analyzing {Path.GetFileName(file.StagedPath)}");
file.Analyze();
});
// Recurse into any archives in this archive
new_files.Where(file => file.IsArchive).Do(file => UpdateArchive(file));
f.FinishedIndexing = true;
if (!_isSyncing)
{
SyncToDisk();
}
Utils.Status("Cleaning Directory");
DeleteDirectory(tmp_dir);
}
private static IEnumerable <string> EnumerateFilesIncludeHidden(string path, IList <string> patterns, bool recursive)
{
// when not checking for hidden directories or files, just enumerate files, which is faster
var fileOptions = baseFileOptions;
if (recursive)
{
fileOptions |= DirectoryEnumerationOptions.Recursive;
}
DirectoryEnumerationFilters fileFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
}
};
bool includeAllFiles = patterns.Count == 0 ||
(patterns.Count == 1 && (patterns[0] == "*.*" || patterns[0] == "*"));
if (!includeAllFiles)
{
fileFilters.InclusionFilter = fsei =>
{
foreach (string pattern in patterns)
{
if (WildcardMatch(fsei.FileName, pattern, true))
{
return(true);
}
else if (pattern == "*.doc" && WildcardMatch(fsei.FileName, "*.doc*", true))
{
return(true);
}
else if (pattern == "*.xls" && WildcardMatch(fsei.FileName, "*.xls*", true))
{
return(true);
}
}
return(false);
};
}
return(Directory.EnumerateFiles(path, fileOptions, fileFilters, PathFormat.FullPath));
}
public void Directory_EnumerateFiles_OK()
{
var dir = Path.GetDirectoryName(typeof(AlphaFSToys).Assembly.Location);
Console.WriteLine($"AlphaFSToys directory name {dir} or {TestContext.CurrentContext.TestDirectory}");
Directory.SetCurrentDirectory(dir);
var dirs = Directory.EnumerateFiles(".").ToArray();
var m = dirs.Contains("bin");
Console.WriteLine(string.Join(",", dirs));
//Assert.That(dirs.Contains(@".\bin")); // NOW get full paths ? hmm.
//Assert.That(dirs.Contains(@".\lib"));
//Assert.That(dirs.Contains(@".\src"));
Assert.That(dirs.Any(x => x.EndsWith(@"AlphaFSTest.pdb")));
Assert.That(dirs.Any(x => x.EndsWith(@"AlphaFSTest.dll")));
}
public void VerifyAllFiles()
{
var skip_files = new HashSet <string> {
"portable.txt"
};
foreach (var dest_file in Directory.EnumerateFiles(InstallFolder, "*", DirectoryEnumerationOptions.Recursive))
{
var rel_file = dest_file.RelativeTo(InstallFolder);
if (rel_file.StartsWith(Consts.LOOTFolderFilesDir) || rel_file.StartsWith(Consts.GameFolderFilesDir))
{
continue;
}
if (!skip_files.Contains(rel_file))
{
Assert.IsTrue(File.Exists(Path.Combine(MO2Folder, rel_file)), $"Only in Destination: {rel_file}");
}
}
var skip_extensions = new HashSet <string> {
".txt", ".ini"
};
foreach (var src_file in Directory.EnumerateFiles(MO2Folder, "*", DirectoryEnumerationOptions.Recursive))
{
var rel_file = src_file.RelativeTo(MO2Folder);
if (rel_file.StartsWith("downloads\\"))
{
continue;
}
var dest_file = Path.Combine(InstallFolder, rel_file);
Assert.IsTrue(File.Exists(dest_file), $"Only in Source: {rel_file}");
var fi_src = new FileInfo(src_file);
var fi_dest = new FileInfo(dest_file);
if (!skip_extensions.Contains(Path.GetExtension(src_file)))
{
Assert.AreEqual(fi_src.Length, fi_dest.Length, $"Differing sizes {rel_file}");
Assert.AreEqual(src_file.FileHash(), dest_file.FileHash(), $"Differing content hash {rel_file}");
}
}
}
/// <summary>
/// Generate a list of immediate children from the current folder
/// </summary>
/// <param name="omitFromScan">Hash representing the hashes that should be skipped</param>
/// <param name="date">True if entry dates should be included, false otherwise (default)</param>
/// <returns>List of BaseFile objects representing the found data</returns>
/// <remarks>TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually</remarks>
public virtual List <BaseFile> GetChildren(Hash omitFromScan = Hash.DeepHashes, bool date = false)
{
if (_children == null || _children.Count == 0)
{
_children = new List <BaseFile>();
foreach (string file in Directory.EnumerateFiles(_filename, "*", SearchOption.TopDirectoryOnly))
{
BaseFile nf = Utilities.GetFileInfo(file, omitFromScan: omitFromScan, date: date);
_children.Add(nf);
}
foreach (string dir in Directory.EnumerateDirectories(_filename, "*", SearchOption.TopDirectoryOnly))
{
Folder fl = new Folder(dir);
_children.Add(fl);
}
}
return(_children);
}
private static void DumpGetAssociation(bool isLocal)
{
Console.WriteLine("\n=== TEST {0} ===", isLocal ? "LOCAL" : "NETWORK");
string path = isLocal ? SysRoot : Path.LocalToUnc(SysRoot);
Console.WriteLine("\nInput Directory Path: [{0}]\n", path);
int cnt = 0;
foreach (string file in Directory.EnumerateFiles(path))
{
string association = Shell32.GetFileAssociation(file);
string contentType = Shell32.GetFileContentType(file);
string defaultIconPath = Shell32.GetFileDefaultIcon(file);
string friendlyAppName = Shell32.GetFileFriendlyAppName(file);
string friendlyDocName = Shell32.GetFileFriendlyDocName(file);
string openWithApp = Shell32.GetFileOpenWithAppName(file);
string verbCommand = Shell32.GetFileVerbCommand(file);
Console.WriteLine("\t#{0:000}\t[{1}]\n", ++cnt, file);
Console.WriteLine("\t\tGetFileAssociation() : [{0}]", association);
Console.WriteLine("\t\tGetFileContentType() : [{0}]", contentType);
Console.WriteLine("\t\tGetFileDefaultIcon() : [{0}]", defaultIconPath);
Console.WriteLine("\t\tGetFileFriendlyAppName(): [{0}]", friendlyAppName);
Console.WriteLine("\t\tGetFileFriendlyDocName(): [{0}]", friendlyDocName);
Console.WriteLine("\t\tGetFileOpenWithAppName(): [{0}]", openWithApp);
Console.WriteLine("\t\tGetFileVerbCommand() : [{0}]", verbCommand);
StopWatcher(true);
Shell32Info shell32Info = Shell32.GetShell32Info(file);
string report = Reporter(true);
string cmd = "print";
verbCommand = shell32Info.GetVerbCommand(cmd);
Console.WriteLine("\n\t\tShell32Info.GetVerbCommand(\"{0}\"): [{1}]", cmd, verbCommand);
Dump(shell32Info, -15);
Console.WriteLine("\n\t{0}\n\n", report);
}
Console.WriteLine("\n");
Assert.IsTrue(cnt > 0, "No entries enumerated.");
}
private void DumpGetAssociation(bool isLocal)
{
Console.WriteLine("\n=== TEST {0} ===", isLocal ? UnitTestConstants.Local : UnitTestConstants.Network);
var path = isLocal ? UnitTestConstants.SysRoot : Path.LocalToUnc(UnitTestConstants.SysRoot);
Console.WriteLine("\nInput Directory Path: [{0}]\n", path);
var cnt = 0;
foreach (var file in Directory.EnumerateFiles(path))
{
var association = Shell32.GetFileAssociation(file);
var contentType = Shell32.GetFileContentType(file);
var defaultIconPath = Shell32.GetFileDefaultIcon(file);
var friendlyAppName = Shell32.GetFileFriendlyAppName(file);
var friendlyDocName = Shell32.GetFileFriendlyDocName(file);
var openWithApp = Shell32.GetFileOpenWithAppName(file);
var verbCommand = Shell32.GetFileVerbCommand(file);
Console.WriteLine("\t#{0:000}\t[{1}]\n", ++cnt, file);
Console.WriteLine("\t\tGetFileAssociation() : [{0}]", association);
Console.WriteLine("\t\tGetFileContentType() : [{0}]", contentType);
Console.WriteLine("\t\tGetFileDefaultIcon() : [{0}]", defaultIconPath);
Console.WriteLine("\t\tGetFileFriendlyAppName(): [{0}]", friendlyAppName);
Console.WriteLine("\t\tGetFileFriendlyDocName(): [{0}]", friendlyDocName);
Console.WriteLine("\t\tGetFileOpenWithAppName(): [{0}]", openWithApp);
Console.WriteLine("\t\tGetFileVerbCommand() : [{0}]", verbCommand);
UnitTestConstants.StopWatcher(true);
var shell32Info = Shell32.GetShell32Info(file);
var report = UnitTestConstants.Reporter(true);
var cmd = "print";
verbCommand = shell32Info.GetVerbCommand(cmd);
Console.WriteLine("\n\t\tShell32Info.GetVerbCommand(\"{0}\"): [{1}]", cmd, verbCommand);
UnitTestConstants.Dump(shell32Info, -15);
Console.WriteLine("\n\t{0}\n\n", report);
}
Console.WriteLine("\n");
Assert.IsTrue(cnt > 0, "No entries enumerated.");
}
private void SetScreenSizeInPrefs()
{
var config = new IniParserConfiguration {
AllowDuplicateKeys = true, AllowDuplicateSections = true
};
foreach (var file in Directory.EnumerateFiles(Path.Combine(OutputFolder, "profiles"), "*refs.ini",
DirectoryEnumerationOptions.Recursive))
{
var parser = new FileIniDataParser(new IniDataParser(config));
var data = parser.ReadFile(file);
if (data.Sections["Display"]["iSize W"] != null && data.Sections["Display"]["iSize H"] != null)
{
data.Sections["Display"]["iSize W"] = SystemParameters.PrimaryScreenWidth.ToString();
data.Sections["Display"]["iSize H"] = SystemParameters.PrimaryScreenHeight.ToString();
}
parser.WriteFile(file, data);
}
}
private static Package CreatePackageFromPath(string path)
{
var package = new Package();
if (!path.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
path += Path.DirectorySeparatorChar;
}
Dictionary <string, string> files = Directory.EnumerateFiles(path, "*.*", SearchOption.AllDirectories)
.ToDictionary(k => k.Replace(path, String.Empty), v => v);
foreach (KeyValuePair <string, string> file in files)
{
FilesystemFileInfo fileInfo = FilesystemFileInfo.CreateFromEntry(file.Value, file.Key);
package.Files.Add(fileInfo);
}
return(package);
}
public void AlphaFS_Shell32_PathFileExists()
{
Console.WriteLine("Filesystem.Shell32.PathFileExists()");
var path = UnitTestConstants.SysRoot;
DumpPathFileExists(path, true);
DumpPathFileExists(Path.LocalToUnc(path), true);
DumpPathFileExists("BlaBlaBla", false);
DumpPathFileExists(Path.Combine(UnitTestConstants.SysRoot, "BlaBlaBla"), false);
var cnt = 0;
UnitTestConstants.StopWatcher(true);
foreach (var file in Directory.EnumerateFiles(UnitTestConstants.SysRoot))
{
var fileExists = Shell32.PathFileExists(file);
Console.WriteLine("\t#{0:000}\tShell32.PathFileExists() == [{1}]: {2}\t\t[{3}]", ++cnt, UnitTestConstants.TextTrue, fileExists, file);
Assert.IsTrue(fileExists);
}
Console.WriteLine("\n\t{0}\n", UnitTestConstants.Reporter(true));
}
public void PathFileExists()
{
Console.WriteLine("Filesystem.Shell32.PathFileExists()");
string path = SysRoot;
DumpPathFileExists(path, true);
DumpPathFileExists(Path.LocalToUnc(path), true);
DumpPathFileExists("BlaBlaBla", false);
DumpPathFileExists(Path.Combine(SysRoot, "BlaBlaBla"), false);
int cnt = 0;
StopWatcher(true);
foreach (string file in Directory.EnumerateFiles(SysRoot))
{
bool fileExists = Shell32.PathFileExists(file);
Console.WriteLine("\t#{0:000}\tShell32.PathFileExists() == [{1}]: {2}\t\t[{3}]", ++cnt, TextTrue, fileExists, file);
Assert.IsTrue(fileExists);
}
Console.WriteLine("\n\t{0}\n", Reporter(true));
}
private static void DeleteDirectory(string path)
{
Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)
.DoProgress("Cleaning VFS Files", file =>
{
try
{
var fi = new FileInfo(file);
fi.Attributes &= ~FileAttributes.ReadOnly;
File.Delete(file);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
});
Directory.EnumerateDirectories(path, DirectoryEnumerationOptions.Recursive)
.OrderByDescending(d => d.Length)
.DoProgress("Cleaning VFS Folders", folder =>
{
try
{
if (!Directory.Exists(folder))
{
return;
}
var di = new DirectoryInfo(folder);
di.Attributes &= ~FileAttributes.ReadOnly;
Directory.Delete(path, true);
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
});
}
public static IList <string> GetGitignoreDirectories(string path, bool recursive, bool followSymlinks)
{
if (File.Exists(Path.Combine(path, ".gitignore")))
{
return new List <string> {
path
}
}
;
var fileOptions = baseFileOptions;
if (recursive)
{
fileOptions |= DirectoryEnumerationOptions.Recursive;
}
if (followSymlinks)
{
fileOptions &= ~DirectoryEnumerationOptions.SkipReparsePoints;
}
List <string> dontRecurseBelow = new List <string>();
DirectoryEnumerationFilters fileFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
},
RecursionFilter = fsei =>
{
if (fsei.IsDirectory && dontRecurseBelow.Any(p => fsei.FullPath.StartsWith(p, StringComparison.CurrentCulture)))
{
return(false);
}
return(true);
},
InclusionFilter = fsei =>
{
if (fsei.FileName == ".gitignore")
{
dontRecurseBelow.Add(Path.GetDirectoryName(fsei.FullPath));
return(true);
}
return(false);
}
};
var list = Directory.EnumerateFiles(path, fileOptions, fileFilters, PathFormat.FullPath)
.Select(s => Path.GetDirectoryName(s)).ToList();
if (list.Count == 0)
{
DirectoryInfo di = new DirectoryInfo(path);
while (di.Parent != null)
{
if (File.Exists(Path.Combine(di.Parent.FullName, ".gitignore")))
{
list.Add(path);
break;
}
di = di.Parent;
}
}
return(list);
}
protected override async Task <ExitCode> Run()
{
var modListPath = (AbsolutePath)Modlist;
if (modListPath.Extension != Consts.ModListExtension && modListPath.FileName != (RelativePath)"modlist.txt")
{
return(CLIUtils.Exit($"The file {Modlist} is not a valid modlist file!", ExitCode.BadArguments));
}
if (Copy && Move)
{
return(CLIUtils.Exit("You can't set both copy and move flags!", ExitCode.BadArguments));
}
var isModlist = modListPath.Extension == Consts.ModListExtension;
var list = new List <TransferFile>();
if (isModlist)
{
ModList modlist;
try
{
modlist = AInstaller.LoadFromFile(modListPath);
}
catch (Exception e)
{
return(CLIUtils.Exit($"Error while loading the Modlist!\n{e}", ExitCode.Error));
}
if (modlist == null)
{
return(CLIUtils.Exit("The Modlist could not be loaded!", ExitCode.Error));
}
CLIUtils.Log($"Modlist contains {modlist.Archives.Count} archives.");
modlist.Archives.Do(a =>
{
var inputPath = Path.Combine(Input, a.Name);
var outputPath = Path.Combine(Output, a.Name);
if (!File.Exists(inputPath))
{
CLIUtils.Log($"File {inputPath} does not exist, skipping.");
return;
}
CLIUtils.Log($"Adding {inputPath} to the transfer list.");
list.Add(new TransferFile(inputPath, outputPath));
var metaInputPath = Path.Combine(inputPath, ".meta");
var metaOutputPath = Path.Combine(outputPath, ".meta");
if (File.Exists(metaInputPath))
{
CLIUtils.Log($"Found meta file {metaInputPath}");
if (IncludeMeta)
{
CLIUtils.Log($"Adding {metaInputPath} to the transfer list.");
list.Add(new TransferFile(metaInputPath, metaOutputPath));
}
else
{
CLIUtils.Log($"Meta file {metaInputPath} will be ignored.");
}
}
else
{
CLIUtils.Log($"Found no meta file for {inputPath}");
if (IncludeMeta)
{
if (string.IsNullOrWhiteSpace(a.Meta))
{
CLIUtils.Log($"Meta for {a.Name} is empty, this should not be possible but whatever.");
return;
}
CLIUtils.Log("Adding meta from archive info the transfer list");
list.Add(new TransferFile(a.Meta, metaOutputPath, true));
}
else
{
CLIUtils.Log($"Meta will be ignored for {a.Name}");
}
}
});
}
else
{
if (!Directory.Exists(Mods))
{
return(CLIUtils.Exit($"Mods directory {Mods} does not exist!", ExitCode.BadArguments));
}
CLIUtils.Log($"Reading modlist.txt from {Modlist}");
string[] modlist = File.ReadAllLines(Modlist);
if (modlist == null || modlist.Length == 0)
{
return(CLIUtils.Exit($"Provided modlist.txt file at {Modlist} is empty or could not be read!", ExitCode.BadArguments));
}
var mods = modlist.Where(s => s.StartsWith("+")).Select(s => s.Substring(1)).ToHashSet();
if (mods.Count == 0)
{
return(CLIUtils.Exit("Counted mods from modlist.txt are 0!", ExitCode.BadArguments));
}
CLIUtils.Log($"Found {mods.Count} mods in modlist.txt");
var downloads = new HashSet <string>();
Directory.EnumerateDirectories(Mods, "*", SearchOption.TopDirectoryOnly)
.Where(d => mods.Contains(Path.GetRelativePath(Path.GetDirectoryName(d), d)))
.Do(d =>
{
var meta = Path.Combine(d, "meta.ini");
if (!File.Exists(meta))
{
CLIUtils.Log($"Mod meta file {meta} does not exist, skipping");
return;
}
string[] ini = File.ReadAllLines(meta);
if (ini == null || ini.Length == 0)
{
CLIUtils.Log($"Mod meta file {meta} could not be read or is empty!");
return;
}
ini.Where(i => !string.IsNullOrWhiteSpace(i) && i.StartsWith("installationFile="))
.Select(i => i.Replace("installationFile=", ""))
.Do(i =>
{
CLIUtils.Log($"Found installationFile {i}");
downloads.Add(i);
});
});
CLIUtils.Log($"Found {downloads.Count} installationFiles from mod metas.");
Directory.EnumerateFiles(Input, "*", SearchOption.TopDirectoryOnly)
.Where(f => downloads.Contains(Path.GetFileNameWithoutExtension(f)))
.Do(f =>
{
CLIUtils.Log($"Found archive {f}");
var outputPath = Path.Combine(Output, Path.GetFileName(f));
CLIUtils.Log($"Adding {f} to the transfer list");
list.Add(new TransferFile(f, outputPath));
var metaInputPath = Path.Combine(f, ".meta");
if (File.Exists(metaInputPath))
{
CLIUtils.Log($"Found meta file for {f} at {metaInputPath}");
if (IncludeMeta)
{
var metaOutputPath = Path.Combine(outputPath, ".meta");
CLIUtils.Log($"Adding {metaInputPath} to the transfer list.");
list.Add(new TransferFile(metaInputPath, metaOutputPath));
}
else
{
CLIUtils.Log("Meta file will be ignored");
}
}
else
{
CLIUtils.Log($"Found no meta file for {f}");
}
});
}
CLIUtils.Log($"Transfer list contains {list.Count} items");
var success = 0;
var failed = 0;
var skipped = 0;
list.Do(f =>
{
if (File.Exists(f.Output))
{
if (Overwrite)
{
CLIUtils.Log($"Output file {f.Output} already exists, it will be overwritten");
if (f.IsMeta || Move)
{
CLIUtils.Log($"Deleting file at {f.Output}");
try
{
File.Delete(f.Output);
}
catch (Exception e)
{
CLIUtils.Log($"Could not delete file {f.Output}!\n{e}");
failed++;
}
}
}
else
{
CLIUtils.Log($"Output file {f.Output} already exists, skipping");
skipped++;
return;
}
}
if (f.IsMeta)
{
CLIUtils.Log($"Writing meta data to {f.Output}");
try
{
File.WriteAllText(f.Output, f.Input, Encoding.UTF8);
success++;
}
catch (Exception e)
{
CLIUtils.Log($"Error while writing meta data to {f.Output}!\n{e}");
failed++;
}
}
else
{
if (Copy)
{
CLIUtils.Log($"Copying file {f.Input} to {f.Output}");
try
{
File.Copy(f.Input, f.Output, Overwrite ? CopyOptions.None : CopyOptions.FailIfExists, CopyMoveProgressHandler, null);
success++;
}
catch (Exception e)
{
CLIUtils.Log($"Error while copying file {f.Input} to {f.Output}!\n{e}");
failed++;
}
}
else if (Move)
{
CLIUtils.Log($"Moving file {f.Input} to {f.Output}");
try
{
File.Move(f.Input, f.Output, Overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None, CopyMoveProgressHandler, null);
success++;
}
catch (Exception e)
{
CLIUtils.Log($"Error while moving file {f.Input} to {f.Output}!\n{e}");
failed++;
}
}
}
});
CLIUtils.Log($"Skipped transfers: {skipped}");
CLIUtils.Log($"Failed transfers: {failed}");
CLIUtils.Log($"Successful transfers: {success}");
return(0);
}
private static IEnumerable <string> EnumerateFilesExcludeHidden(string path, IList <string> patterns, bool recursive)
{
// when checking for hidden directories, enumerate the directories separately from files to check for hidden flag on directories
DirectoryInfo di = new DirectoryInfo(path);
// the root of the drive has the hidden attribute set, so don't stop on this hidden directory
if (di.Attributes.HasFlag(FileAttributes.Hidden) && (di.Root != di))
{
yield break;
}
var dirOptions = baseDirOptions;
if (recursive)
{
dirOptions |= DirectoryEnumerationOptions.Recursive;
}
DirectoryEnumerationFilters dirFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
}
};
dirFilters.InclusionFilter = fsei =>
{
return(!fsei.IsHidden);
};
DirectoryEnumerationFilters fileFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
}
};
bool includeAllFiles = patterns.Count == 0 ||
(patterns.Count == 1 && (patterns[0] == "*.*" || patterns[0] == "*"));
if (includeAllFiles)
{
fileFilters.InclusionFilter = fsei =>
{
return(!fsei.IsHidden);
};
}
else
{
fileFilters.InclusionFilter = fsei =>
{
if (fsei.IsHidden)
{
return(false);
}
foreach (string pattern in patterns)
{
if (WildcardMatch(fsei.FileName, pattern, true))
{
return(true);
}
else if (pattern == "*.doc" && WildcardMatch(fsei.FileName, "*.doc*", true))
{
return(true);
}
else if (pattern == "*.xls" && WildcardMatch(fsei.FileName, "*.xls*", true))
{
return(true);
}
}
return(false);
};
}
IEnumerable <string> directories = new string[] { path };
if (recursive)
{
directories = directories.Concat(Directory.EnumerateDirectories(path, dirOptions, dirFilters, PathFormat.FullPath));
}
foreach (var directory in directories)
{
IEnumerable <string> matches = Directory.EnumerateFiles(directory, baseFileOptions, fileFilters, PathFormat.FullPath);
foreach (var file in matches)
{
yield return(file);
}
}
}
private static IEnumerable <string> EnumerateFilesImpl(string path, IList <string> patterns,
FileFilter filter, Gitignore gitignore)
{
DirectoryEnumerationFilters fileFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
}
};
bool includeAllFiles = (patterns.Count == 0 ||
(patterns.Count == 1 && (patterns[0] == "*.*" || patterns[0] == "*"))) &&
(gitignore == null || gitignore.Files.Count == 0);
if (includeAllFiles)
{
fileFilters.InclusionFilter = fsei =>
{
if (!filter.IncludeHidden && fsei.IsHidden)
{
return(false);
}
return(true);
};
}
else
{
fileFilters.InclusionFilter = fsei =>
{
if (!filter.IncludeHidden && fsei.IsHidden)
{
return(false);
}
if (gitignore != null && gitignore.Files.Contains(fsei.FullPath))
{
return(false);
}
foreach (string pattern in patterns)
{
if (WildcardMatch(fsei.FileName, pattern, true))
{
return(true);
}
}
if (filter.IncludeArchive)
{
foreach (string pattern in ArchiveDirectory.Patterns)
{
if (WildcardMatch(fsei.FileName, pattern, true))
{
return(true);
}
}
}
return(false);
};
}
var fileOptions = baseFileOptions;
if (filter.FollowSymlinks)
{
fileOptions &= ~DirectoryEnumerationOptions.SkipReparsePoints;
}
return(Directory.EnumerateFiles(path, fileOptions, fileFilters, PathFormat.FullPath));
}
public static IErrorResponse CheckValidInstallPath(string path, string downloadFolder)
{
var ret = Utils.IsDirectoryPathValid(path);
if (!ret.Succeeded)
{
return(ret);
}
if (!Directory.Exists(path))
{
return(ErrorResponse.Success);
}
// Check folder does not have a Wabbajack ModList
foreach (var file in Directory.EnumerateFiles(path))
{
if (!File.Exists(file))
{
continue;
}
if (System.IO.Path.GetExtension(file).Equals(Consts.ModListExtension))
{
return(ErrorResponse.Fail($"Cannot install into a folder with a Wabbajack ModList inside of it"));
}
}
// Check folder is either empty, or a likely valid previous install
if (!Directory.IsEmpty(path))
{
// If we have a MO2 install, assume good to go
if (Directory.EnumerateFiles(path).Any(file =>
{
var fileName = Path.GetFileName(file);
if (fileName.Equals("ModOrganizer.exe", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
if (fileName.Equals("ModOrganizer.ini", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
return(false);
}))
{
return(ErrorResponse.Success);
}
// If we don't have a MO2 install, and there's any file that's not in the downloads folder, mark failure
if (Directory.EnumerateFiles(path).Any(file =>
{
var fileName = Path.GetFileName(file);
if (string.IsNullOrWhiteSpace(downloadFolder))
{
return(true);
}
return(!Utils.IsUnderneathDirectory(file, downloadFolder));
}))
{
return(ErrorResponse.Fail($"Cannot install into a non-empty folder that does not look like a previous WJ installation.\n" +
$"To override, delete all installed files from your target installation folder. Any files in your download folder are okay to keep."));
}
}
return(ErrorResponse.Success);
}
/// <summary>
/// The user may already have some files in the OutputFolder. If so we can go through these and
/// figure out which need to be updated, deleted, or left alone
/// </summary>
public async Task OptimizeModlist()
{
Utils.Log("Optimizing ModList directives");
// Clone the ModList so our changes don't modify the original data
ModList = ModList.Clone();
var indexed = ModList.Directives.ToDictionary(d => d.To);
UpdateTracker.NextStep("Looking for files to delete");
await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
.PMap(Queue, UpdateTracker, f =>
{
var relative_to = f.RelativeTo(OutputFolder);
Utils.Status($"Checking if ModList file {relative_to}");
if (indexed.ContainsKey(relative_to) || f.IsInPath(DownloadFolder))
{
return;
}
Utils.Log($"Deleting {relative_to} it's not part of this ModList");
File.Delete(f);
});
UpdateTracker.NextStep("Looking for unmodified files");
(await indexed.Values.PMap(Queue, UpdateTracker, d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
Status($"Optimizing {d.To}");
var path = Path.Combine(OutputFolder, d.To);
if (!File.Exists(path))
{
return(null);
}
var fi = new FileInfo(path);
if (fi.Length != d.Size)
{
return(null);
}
return(path.FileHash() == d.Hash ? d : null);
}))
.Where(d => d != null)
.Do(d => indexed.Remove(d.To));
Utils.Log("Cleaning empty folders");
var expectedFolders = indexed.Keys
// We ignore the last part of the path, so we need a dummy file name
.Append(Path.Combine(DownloadFolder, "_"))
.SelectMany(path =>
{
// Get all the folders and all the folder parents
// so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"]
var split = path.Split('\\');
return(Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))));
}).Distinct()
.Select(p => Path.Combine(OutputFolder, p))
.ToHashSet();
try
{
Directory.EnumerateDirectories(OutputFolder, DirectoryEnumerationOptions.Recursive)
.Where(p => !expectedFolders.Contains(p))
.OrderByDescending(p => p.Length)
.Do(Utils.DeleteDirectory);
}
catch (Exception)
{
// ignored because it's not worth throwing a fit over
Utils.Log("Error when trying to clean empty folders. This doesn't really matter.");
}
UpdateTracker.NextStep("Updating ModList");
Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
var requiredArchives = indexed.Values.OfType <FromArchive>()
.GroupBy(d => d.ArchiveHashPath[0])
.Select(d => d.Key)
.ToHashSet();
ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();
ModList.Directives = indexed.Values.ToList();
}
private void DumpGetSetAttributes(bool isLocal)
{
Console.WriteLine("\n=== TEST {0} ===", isLocal ? UnitTestConstants.Local : UnitTestConstants.Network);
var tmp = Path.Combine(Path.GetTempPath(), "File.SetAttributes()-" + Path.GetRandomFileName());
var tempPath = isLocal ? tmp : Path.LocalToUnc(tmp);
var sys32 = isLocal ? UnitTestConstants.SysRoot32 : Path.LocalToUnc(UnitTestConstants.SysRoot32);
Console.WriteLine("\nInput Path: [{0}]", sys32);
// Just enumerate and compare attributes in folder: C:\Windows\System32
foreach (var file in Directory.EnumerateFiles(sys32))
{
var actual = File.GetAttributes(file);
var expected = System.IO.File.GetAttributes(file);
Assert.AreEqual(expected, actual, "AlphaFS != System.IO");
}
Console.WriteLine("\nInput Path: [{0}]", tempPath);
// Create some folders and files.
UnitTestConstants.CreateDirectoriesAndFiles(tempPath, 10, true);
var apply = FileAttributes.Hidden | FileAttributes.Archive | FileAttributes.System | FileAttributes.ReadOnly;
Console.WriteLine("\nSetAttributes(): [{0}]", apply);
var allOk = true;
var cnt = 0;
UnitTestConstants.StopWatcher(true);
foreach (var file in Directory.EnumerateFiles(tempPath))
{
try
{
File.SetAttributes(file, apply);
var actual = File.GetAttributes(file);
var expected = System.IO.File.GetAttributes(file);
Console.WriteLine("\n\t#{0:000}\tFile : [{1}]\n\t\tAlphaFS : [{2}]\n\t\tSystem.IO: [{3}]", ++cnt, file, expected, actual);
if (cnt == 0)
{
Assert.Inconclusive("Nothing was enumerated, but it was expected.");
}
Assert.AreEqual(expected, actual, "AlphaFS != System.IO");
}
catch (Exception ex)
{
allOk = false;
Console.WriteLine("\n\tCaught (unexpected) {0}: [{1}]", ex.GetType().FullName, ex.Message.Replace(Environment.NewLine, " "));
}
}
Console.WriteLine();
Console.WriteLine(UnitTestConstants.Reporter());
Assert.IsTrue(allOk);
apply = FileAttributes.Normal;
Console.WriteLine("\nSetAttributes(): [{0}]", apply);
allOk = true;
cnt = 0;
UnitTestConstants.StopWatcher(true);
foreach (var file in Directory.EnumerateFiles(tempPath))
{
try
{
File.SetAttributes(file, apply);
var actual = File.GetAttributes(file);
var expected = System.IO.File.GetAttributes(file);
Console.WriteLine("\n\t#{0:000}\tFile : [{1}]\n\t\tAlphaFS : [{2}]\n\t\tSystem.IO: [{3}]", ++cnt, file, expected, actual);
if (cnt == 0)
{
Assert.Inconclusive("Nothing was enumerated, but it was expected.");
}
Assert.AreEqual(expected, actual, "AlphaFS != System.IO");
}
catch (Exception ex)
{
allOk = false;
Console.WriteLine("\n\tCaught (unexpected) {0}: [{1}]", ex.GetType().FullName, ex.Message.Replace(Environment.NewLine, " "));
}
}
Console.WriteLine();
Console.WriteLine(UnitTestConstants.Reporter());
Directory.Delete(tempPath, true);
Assert.IsFalse(Directory.Exists(tempPath), "Cleanup failed: Directory should have been removed.");
Assert.IsTrue(allOk);
Console.WriteLine();
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("Kruacm8p1B6RFAw2WMnKcEqkQcnWRkF3RmPSOzlW");
SetupNLog();
SetupPatterns();
_logger = LogManager.GetCurrentClassLogger();
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to search. Either this or -d is required");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to recursively process. Either this or -f is required");
_fluentCommandLineParser.Setup(arg => arg.SaveTo)
.As('o')
.WithDescription("File to save results to");
_fluentCommandLineParser.Setup(arg => arg.GetAscii)
.As('a')
.SetDefault(true)
.WithDescription("If set, look for ASCII strings. Default is true. Use -a false to disable");
_fluentCommandLineParser.Setup(arg => arg.GetUnicode)
.As('u')
.SetDefault(true)
.WithDescription("If set, look for Unicode strings. Default is true. Use -u false to disable");
_fluentCommandLineParser.Setup(arg => arg.MinimumLength)
.As('m').SetDefault(3).WithDescription("Minimum string length. Default is 3");
_fluentCommandLineParser.Setup(arg => arg.BlockSizeMb)
.As('b').SetDefault(512).WithDescription("Chunk size in MB. Valid range is 1 to 1024. Default is 512");
_fluentCommandLineParser.Setup(arg => arg.Quiet)
.As('q').SetDefault(false).WithDescription("Quiet mode (Do not show header or total number of hits)");
_fluentCommandLineParser.Setup(arg => arg.QuietQuiet)
.As('s')
.SetDefault(false)
.WithDescription(
"Really Quiet mode (Do not display hits to console. Speeds up processing when using -o)");
_fluentCommandLineParser.Setup(arg => arg.MaximumLength)
.As('x').SetDefault(-1).WithDescription("Maximum string length. Default is unlimited\r\n");
_fluentCommandLineParser.Setup(arg => arg.GetPatterns)
.As('p').SetDefault(false).WithDescription("Display list of built in regular expressions");
_fluentCommandLineParser.Setup(arg => arg.LookForString)
.As("ls")
.SetDefault(string.Empty)
.WithDescription("String to look for. When set, only matching strings are returned");
_fluentCommandLineParser.Setup(arg => arg.LookForRegex)
.As("lr")
.SetDefault(string.Empty)
.WithDescription("Regex to look for. When set, only strings matching the regex are returned");
_fluentCommandLineParser.Setup(arg => arg.StringFile)
.As("fs")
.SetDefault(string.Empty)
.WithDescription("File containing strings to look for. When set, only matching strings are returned");
_fluentCommandLineParser.Setup(arg => arg.RegexFile)
.As("fr")
.SetDefault(string.Empty)
.WithDescription(
"File containing regex patterns to look for. When set, only strings matching regex patterns are returned\r\n");
_fluentCommandLineParser.Setup(arg => arg.AsciiRange)
.As("ar")
.SetDefault("[\x20-\x7E]")
.WithDescription(
@"Range of characters to search for in 'Code page' strings. Specify as a range of characters in hex format and enclose in quotes. Default is [\x20 -\x7E]");
_fluentCommandLineParser.Setup(arg => arg.UnicodeRange)
.As("ur")
.SetDefault("[\u0020-\u007E]")
.WithDescription(
"Range of characters to search for in Unicode strings. Specify as a range of characters in hex format and enclose in quotes. Default is [\\u0020-\\u007E]\r\n");
_fluentCommandLineParser.Setup(arg => arg.CodePage)
.As("cp")
.SetDefault(1252)
.WithDescription(
"Code page to use. Default is 1252. Use the Identifier value for code pages at https://goo.gl/ig6DxW");
_fluentCommandLineParser.Setup(arg => arg.FileMask)
.As("mask")
.SetDefault(string.Empty)
.WithDescription(
"When using -d, file mask to search for. * and ? are supported. This option has no effect when using -f");
_fluentCommandLineParser.Setup(arg => arg.RegexOnly)
.As("ro")
.SetDefault(false)
.WithDescription(
"When true, list the string matched by regex pattern vs string the pattern was found in (This may result in duplicate strings in output. ~ denotes approx. offset)");
_fluentCommandLineParser.Setup(arg => arg.ShowOffset)
.As("off")
.SetDefault(false)
.WithDescription(
$"Show offset to hit after string, followed by the encoding (A={_fluentCommandLineParser.Object.CodePage}, U=Unicode)\r\n");
_fluentCommandLineParser.Setup(arg => arg.SortAlpha)
.As("sa").SetDefault(false).WithDescription("Sort results alphabetically");
_fluentCommandLineParser.Setup(arg => arg.SortLength)
.As("sl").SetDefault(false).WithDescription("Sort results by length");
var header =
$"bstrings version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/bstrings";
var footer = @"Examples: bstrings.exe -f ""C:\Temp\UsrClass 1.dat"" --ls URL" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someFile.txt"" --lr guid" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\aBigFile.bin"" --fs c:\temp\searchStrings.txt --fr c:\temp\searchRegex.txt -s" +
"\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --mask ""*.dll""" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --ar ""[\x20-\x37]""" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --cp 10007" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --ls test" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someOtherFile.txt"" --lr cc -sa" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someOtherFile.txt"" --lr cc -sa -m 15 -x 22" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\UsrClass 1.dat"" --ls mui -sl" + "\r\n\t ";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (_fluentCommandLineParser.Object.GetPatterns)
{
_logger.Warn("Name \t\tDescription");
foreach (var regExPattern in RegExPatterns.OrderBy(t => t.Key))
{
var desc = RegExDesc[regExPattern.Key];
_logger.Info($"{regExPattern.Key}\t{desc}");
}
_logger.Info("");
_logger.Info("To use a built in pattern, supply the Name to the --lr switch\r\n");
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("Either -f or -d is required. Exiting");
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false &&
!File.Exists(_fluentCommandLineParser.Object.File) &&
_fluentCommandLineParser.Object.FileMask.Length == 0)
{
_logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting");
return;
}
if (_fluentCommandLineParser.Object.Directory.IsNullOrEmpty() == false &&
!Directory.Exists(_fluentCommandLineParser.Object.Directory) &&
_fluentCommandLineParser.Object.FileMask.Length == 0)
{
_logger.Warn($"Directory '{_fluentCommandLineParser.Object.Directory}' not found. Exiting");
return;
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(header);
_logger.Info("");
}
var files = new List <string>();
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
files.Add(Path.GetFullPath(_fluentCommandLineParser.Object.File));
}
else
{
try
{
if (_fluentCommandLineParser.Object.FileMask.Length > 0)
{
files.AddRange(Directory.EnumerateFiles(Path.GetFullPath(_fluentCommandLineParser.Object.Directory),
_fluentCommandLineParser.Object.FileMask, SearchOption.AllDirectories));
}
else
{
files.AddRange(Directory.EnumerateFiles(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), "*",
SearchOption.AllDirectories));
}
}
catch (Exception ex)
{
_logger.Error(
$"Error getting files in '{_fluentCommandLineParser.Object.Directory}'. Error message: {ex.Message}");
return;
}
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Command line: {string.Join(" ", args)}");
_logger.Info("");
}
StreamWriter sw = null;
var globalCounter = 0;
var globalHits = 0;
double globalTimespan = 0;
var withBoundaryHits = false;
if (_fluentCommandLineParser.Object.SaveTo.IsNullOrEmpty() == false && _fluentCommandLineParser.Object.SaveTo.Length > 0)
{
_fluentCommandLineParser.Object.SaveTo = _fluentCommandLineParser.Object.SaveTo.TrimEnd('\\');
var dir = Path.GetDirectoryName(_fluentCommandLineParser.Object.SaveTo);
if (dir != null && Directory.Exists(dir) == false)
{
try
{
Directory.CreateDirectory(dir);
}
catch (Exception)
{
_logger.Warn(
$"Invalid path: '{_fluentCommandLineParser.Object.SaveTo}'. Results will not be saved to a file.");
_logger.Info("");
_fluentCommandLineParser.Object.SaveTo = string.Empty;
}
}
else
{
if (dir == null)
{
_logger.Warn($"Invalid path: '{_fluentCommandLineParser.Object.SaveTo}");
_fluentCommandLineParser.Object.SaveTo = string.Empty;
}
}
if (_fluentCommandLineParser.Object.SaveTo.Length > 0 && !_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Saving hits to '{_fluentCommandLineParser.Object.SaveTo}'");
_logger.Info("");
}
if (_fluentCommandLineParser.Object.SaveTo.Length > 0)
{
// try
// {
// File.Create(_fluentCommandLineParser.Object.SaveTo);
// }
// catch (Exception e)
// {
// _logger.Fatal($"Unable to create output file '{_fluentCommandLineParser.Object.SaveTo}'! Check permissions and try again! Error: {e.Message}");
// return;
// }
sw = new StreamWriter(_fluentCommandLineParser.Object.SaveTo, true);
}
}
foreach (var file in files)
{
if (File.Exists(file) == false)
{
_logger.Warn($"'{file}' does not exist! Skipping");
}
_sw = new Stopwatch();
_sw.Start();
var counter = 0;
var hits = new HashSet <string>();
var regPattern = _fluentCommandLineParser.Object.LookForRegex;
if (RegExPatterns.ContainsKey(_fluentCommandLineParser.Object.LookForRegex))
{
regPattern = RegExPatterns[_fluentCommandLineParser.Object.LookForRegex];
}
if (regPattern.Length > 0 && !_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Searching via RegEx pattern: {regPattern}");
_logger.Info("");
}
var minLength = 3;
if (_fluentCommandLineParser.Object.MinimumLength > 0)
{
minLength = _fluentCommandLineParser.Object.MinimumLength;
}
var maxLength = -1;
if (_fluentCommandLineParser.Object.MaximumLength > minLength)
{
maxLength = _fluentCommandLineParser.Object.MaximumLength;
}
var chunkSizeMb = _fluentCommandLineParser.Object.BlockSizeMb < 1 ||
_fluentCommandLineParser.Object.BlockSizeMb > 1024
? 512
: _fluentCommandLineParser.Object.BlockSizeMb;
var chunkSizeBytes = chunkSizeMb * 1024 * 1024;
var fileSizeBytes = new FileInfo(file).Length;
var bytesRemaining = fileSizeBytes;
long offset = 0;
var chunkIndex = 1;
var totalChunks = fileSizeBytes / chunkSizeBytes + 1;
var hsuffix = totalChunks == 1 ? "" : "s";
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
$"Searching {totalChunks:N0} chunk{hsuffix} ({chunkSizeMb} MB each) across {GetSizeReadable(fileSizeBytes)} in '{file}'");
_logger.Info("");
}
try
{
MappedStream ms = null;
try
{
var fs =
File.Open(File.GetFileSystemEntryInfo(file).LongFullPath, FileMode.Open, FileAccess.Read,
FileShare.Read, PathFormat.LongFullPath);
ms = MappedStream.FromStream(fs, Ownership.None);
}
catch (Exception)
{
_logger.Warn($"Unable to open file directly. This usually means the file is in use. Switching to raw access\r\n");
}
if (ms == null)
{
//raw mode
var ss = OpenFile(file);
ms = MappedStream.FromStream(ss, Ownership.None);
}
using (ms)
{
while (bytesRemaining > 0)
{
if (bytesRemaining <= chunkSizeBytes)
{
chunkSizeBytes = (int)bytesRemaining;
}
var chunk = new byte[chunkSizeBytes];
ms.Read(chunk, 0, chunkSizeBytes);
if (_fluentCommandLineParser.Object.GetUnicode)
{
var uh = GetUnicodeHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in uh)
{
hits.Add(h);
}
}
if (_fluentCommandLineParser.Object.GetAscii)
{
var ah = GetAsciiHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in ah)
{
hits.Add(h);
}
}
offset += chunkSizeBytes;
bytesRemaining -= chunkSizeBytes;
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
$"Chunk {chunkIndex:N0} of {totalChunks:N0} finished. Total strings so far: {hits.Count:N0} Elapsed time: {_sw.Elapsed.TotalSeconds:N3} seconds. Average strings/sec: {hits.Count/_sw.Elapsed.TotalSeconds:N0}");
}
chunkIndex += 1;
}
//do chunk boundary checks to make sure we get everything and not split things
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
"Primary search complete. Looking for strings across chunk boundaries...");
}
bytesRemaining = fileSizeBytes;
chunkSizeBytes = chunkSizeMb * 1024 * 1024;
offset = chunkSizeBytes - _fluentCommandLineParser.Object.MinimumLength * 10 * 2;
//move starting point backwards for our starting point
chunkIndex = 0;
var boundaryChunkSize = _fluentCommandLineParser.Object.MinimumLength * 10 * 2 * 2;
//grab the same # of bytes on both sides of the boundary
while (bytesRemaining > 0)
{
if (offset + boundaryChunkSize > fileSizeBytes)
{
break;
}
var chunk = new byte[boundaryChunkSize];
ms.Read(chunk, 0, boundaryChunkSize);
if (_fluentCommandLineParser.Object.GetUnicode)
{
var uh = GetUnicodeHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in uh)
{
hits.Add(" " + h);
}
if (withBoundaryHits == false && uh.Count > 0)
{
withBoundaryHits = uh.Count > 0;
}
}
if (_fluentCommandLineParser.Object.GetAscii)
{
var ah = GetAsciiHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in ah)
{
hits.Add(" " + h);
}
if (withBoundaryHits == false && ah.Count > 0)
{
withBoundaryHits = true;
}
}
offset += chunkSizeBytes;
bytesRemaining -= chunkSizeBytes;
chunkIndex += 1;
}
}
}
catch (Exception ex)
{
_logger.Info("");
_logger.Error($"Error: {ex.Message}");
}
_sw.Stop();
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("Search complete.");
_logger.Info("");
}
if (_fluentCommandLineParser.Object.SortAlpha)
{
_logger.Info("Sorting alphabetically...");
_logger.Info("");
var tempList = hits.ToList();
tempList.Sort();
hits = new HashSet <string>(tempList);
}
else if (_fluentCommandLineParser.Object.SortLength)
{
_logger.Info("Sorting by length...");
_logger.Info("");
var tempList = SortByLength(hits.ToList()).ToList();
hits = new HashSet <string>(tempList);
}
var fileStrings = new HashSet <string>();
var regexStrings = new HashSet <string>();
//set up highlighting
if (_fluentCommandLineParser.Object.LookForString.Length > 0)
{
fileStrings.Add(_fluentCommandLineParser.Object.LookForString);
}
if (_fluentCommandLineParser.Object.LookForRegex.Length > 0)
{
regexStrings.Add(regPattern);
}
if (_fluentCommandLineParser.Object.StringFile.IsNullOrEmpty() == false || _fluentCommandLineParser.Object.RegexFile.IsNullOrEmpty() == false)
{
if (_fluentCommandLineParser.Object.StringFile.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.StringFile))
{
fileStrings.UnionWith(new HashSet <string>(File.ReadAllLines(_fluentCommandLineParser.Object.StringFile)));
}
else
{
_logger.Error($"Strings file '{_fluentCommandLineParser.Object.StringFile}' not found.");
}
}
if (_fluentCommandLineParser.Object.RegexFile.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.RegexFile))
{
regexStrings.UnionWith(new HashSet <string>(File.ReadAllLines(_fluentCommandLineParser.Object.RegexFile)));
}
else
{
_logger.Error($"Regex file '{_fluentCommandLineParser.Object.RegexFile}' not found.");
}
}
}
AddHighlightingRules(fileStrings.ToList());
if (_fluentCommandLineParser.Object.RegexOnly == false)
{
AddHighlightingRules(regexStrings.ToList(), true);
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("Processing strings...");
_logger.Info("");
}
foreach (var hit in hits)
{
if (hit.Length == 0)
{
continue;
}
if (fileStrings.Count > 0 || regexStrings.Count > 0)
{
foreach (var fileString in fileStrings)
{
if (fileString.Trim().Length == 0)
{
continue;
}
if (hit.IndexOf(fileString, StringComparison.InvariantCultureIgnoreCase) < 0)
{
continue;
}
counter += 1;
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
var hitoffset = "";
if (_fluentCommandLineParser.Object.ShowOffset)
{
hitoffset = $"~{hit.Split('\t').Last()}";
}
foreach (var regString in regexStrings)
{
if (regString.Trim().Length == 0)
{
continue;
}
try
{
var reg1 = new Regex(regString,
RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
if (reg1.IsMatch(hit) == false)
{
continue;
}
counter += 1;
if (_fluentCommandLineParser.Object.RegexOnly)
{
foreach (var match in reg1.Matches(hit))
{
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info($"{match}\t{hitoffset}");
}
sw?.WriteLine($"{match}\t{hitoffset}");
}
}
else
{
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
}
catch (Exception ex)
{
_logger.Error($"Error setting up regular expression '{regString}': {ex.Message}");
}
}
}
else
{
//dump all strings
counter += 1;
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
}
if (_fluentCommandLineParser.Object.Quiet)
{
continue;
}
var suffix = counter == 1 ? "" : "s";
_logger.Info("");
if (withBoundaryHits)
{
_logger.Info("** Strings prefixed with 2 spaces are hits found across chunk boundaries **");
_logger.Info("");
}
_logger.Info(
$"Found {counter:N0} string{suffix} in {_sw.Elapsed.TotalSeconds:N3} seconds. Average strings/sec: {hits.Count/_sw.Elapsed.TotalSeconds:N0}");
globalCounter += counter;
globalHits += hits.Count;
globalTimespan += _sw.Elapsed.TotalSeconds;
if (files.Count > 1)
{
_logger.Info(
"-------------------------------------------------------------------------------------\r\n");
}
}
if (sw != null)
{
sw.Flush();
sw.Close();
}
if (!_fluentCommandLineParser.Object.Quiet && files.Count > 1)
{
var suffix = globalCounter == 1 ? "" : "s";
_logger.Info("");
_logger.Info(
$"Found {globalCounter:N0} string{suffix} in {globalTimespan:N3} seconds. Average strings/sec: {globalHits/globalTimespan:N0}");
}
}
public static IList <string> GetGitignoreDirectories(string path, bool recursive, bool followSymlinks)
{
if (File.Exists(Path.Combine(path, ".gitignore")))
{
return new List <string> {
path
}
}
;
var fileOptions = baseFileOptions;
if (recursive)
{
fileOptions |= DirectoryEnumerationOptions.Recursive;
}
if (followSymlinks)
{
fileOptions &= ~DirectoryEnumerationOptions.SkipReparsePoints;
}
List <string> dontRecurseBelow = new List <string>
{
@"C:\$Recycle.Bin"
};
foreach (var sf in new[]
{
Environment.SpecialFolder.Windows,
Environment.SpecialFolder.ProgramFiles,
Environment.SpecialFolder.ProgramFilesX86,
})
{
string p = Environment.GetFolderPath(sf);
if (!string.IsNullOrEmpty(p))
{
dontRecurseBelow.Add(p);
}
}
DirectoryEnumerationFilters fileFilters = new DirectoryEnumerationFilters
{
ErrorFilter = (errorCode, errorMessage, pathProcessed) =>
{
logger.Error($"Find file error {errorCode}: {errorMessage} on {pathProcessed}");
return(true);
},
RecursionFilter = fsei =>
{
if (Utils.CancelSearch)
{
throw new OperationCanceledException();
}
if (fsei.IsDirectory && dontRecurseBelow.Any(p => fsei.FullPath.StartsWith(p, true, CultureInfo.CurrentCulture)))
{
return(false);
}
return(true);
},
InclusionFilter = fsei =>
{
if (Utils.CancelSearch)
{
throw new OperationCanceledException();
}
if (fsei.FileName == ".gitignore")
{
dontRecurseBelow.Add(Path.GetDirectoryName(fsei.FullPath));
return(true);
}
return(false);
}
};
try
{
// search down subdirectories
var list = Directory.EnumerateFiles(path, fileOptions, fileFilters, PathFormat.FullPath)
.Select(s => Path.GetDirectoryName(s)).ToList();
if (list.Count == 0)
{
// not found, search up the tree
DirectoryInfo di = new DirectoryInfo(path);
while (di.Parent != null)
{
if (File.Exists(Path.Combine(di.Parent.FullName, ".gitignore")))
{
list.Add(path);
break;
}
di = di.Parent;
}
}
return(list);
}
catch (OperationCanceledException)
{
return(new List <string>());
}
}
