private static void ExtractAllWithBSA(string source, string dest)
{
try
{
using (var arch = BSADispatch.OpenRead(source))
{
arch.Files.PMap(f =>
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
{
path = f.Path.Substring(1);
}
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent))
{
Directory.CreateDirectory(parent);
}
using (var fs = File.OpenWrite(out_path))
{
f.CopyDataTo(fs);
}
});
}
}
catch (Exception ex)
{
Utils.Log($"While Extracting {source}");
throw ex;
}
}
public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output)
{
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch)
VALUES (@fromHash, @toHash, @patchSize, @patch)";
var dataA = a.xxHash();
var dataB = b.xxHash();
cmd.Parameters.AddWithValue("@fromHash", (long)dataA);
cmd.Parameters.AddWithValue("@toHash", (long)dataB);
await using var patch = new MemoryStream();
Utils.Status("Creating Patch");
OctoDiff.Create(a, b, patch);
patch.Position = 0;
cmd.Parameters.AddWithValue("@patchSize", patch.Length);
cmd.Parameters.AddWithValue("@patch", patch.ToArray());
try
{
await cmd.ExecuteNonQueryAsync();
}
catch (SQLiteException ex)
{
if (!ex.Message.StartsWith("constraint failed"))
{
throw;
}
}
await patch.CopyToAsync(output);
}
private void UpdateStatus()
{
if (_inner.Length != 0)
{
Utils.Status(_message, (int)(_inner.Position * 100 / _inner.Length));
}
}
private static async Task ExtractAllWithBSA(WorkQueue queue, AbsolutePath source, AbsolutePath dest)
{
try
{
using var arch = BSADispatch.OpenRead(source);
await arch.Files
.PMap(queue, f =>
{
Utils.Status($"Extracting {(string)f.Path}");
var outPath = f.Path.RelativeTo(dest);
var parent = outPath.Parent;
if (!parent.IsDirectory)
{
parent.CreateDirectory();
}
using var fs = outPath.Create();
f.CopyDataTo(fs);
});
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
}
}
private static void ExtractAllWithBSA(string source, string dest)
{
using (var arch = new BSAReader(source))
{
arch.Files.PMap(f =>
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
{
path = f.Path.Substring(1);
}
Utils.Status($"Extracting {path}");
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent))
{
Directory.CreateDirectory(parent);
}
using (var fs = File.OpenWrite(out_path))
{
f.CopyDataTo(fs);
}
});
}
}
private static async Task ExtractAllWithBSA(WorkQueue queue, string source, string dest)
{
try
{
using (var arch = BSADispatch.OpenRead(source))
{
await arch.Files
.PMap(queue, f =>
{
var path = f.Path;
if (f.Path.StartsWith("\\"))
{
path = f.Path.Substring(1);
}
Utils.Status($"Extracting {path}");
var outPath = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(outPath);
if (!Directory.Exists(parent))
{
Directory.CreateDirectory(parent);
}
using (var fs = File.Open(outPath, System.IO.FileMode.Create))
{
f.CopyDataTo(fs);
}
});
}
}
catch (Exception ex)
{
Utils.ErrorThrow(ex, $"While Extracting {source}");
}
}
private static void ExtractAllWithInno(string source, string dest)
{
Utils.Log($"Extracting {Path.GetFileName(source)}");
var info = new ProcessStartInfo
{
FileName = "innounp.exe",
Arguments = $"-x -y -b -d\"{dest}\" \"{source}\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var p = new Process
{
StartInfo = info
};
p.Start();
ChildProcessTracker.AddProcess(p);
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
}
var name = Path.GetFileName(source);
try
{
while (!p.HasExited)
{
var line = p.StandardOutput.ReadLine();
if (line == null)
{
break;
}
var percent = 0;
if (line.Length > 4 && line[3] == '%')
{
int.TryParse(line.Substring(0, 3), out percent);
Utils.Status($"Extracting {name} - {line.Trim()}", percent);
}
}
}
catch (Exception ex)
{
}
p.WaitForExit();
if (p.ExitCode != 0)
{
Utils.Log(p.StandardOutput.ReadToEnd());
Utils.Log($"Extraction error extracting {source}");
}
}
public static async Task <long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash,
Stream?patchOutStream = null)
{
var key = PatchKey(srcHash, destHash);
var patch = _patchCache !.Get(key);
if (patch != null)
{
if (patchOutStream == null)
{
return(patch.Length);
}
await patchOutStream.WriteAsync(patch);
return(patch.Length);
}
Utils.Status("Creating Patch");
await using var sigStream = new MemoryStream();
await using var patchStream = new MemoryStream();
OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
_patchCache.Put(key, patchStream.ToArray());
if (patchOutStream == null)
{
return(patchStream.Position);
}
patchStream.Position = 0;
await patchStream.CopyToAsync(patchOutStream);
return(patchStream.Position);
}
private static void CreateSignature(Stream oldData, FileStream sigStream)
{
Utils.Status("Creating Patch Signature");
var signatureBuilder = new SignatureBuilder();
signatureBuilder.Build(oldData, new SignatureWriter(sigStream));
sigStream.Position = 0;
}
public static void WaitForExitAndWarn(this Process process, TimeSpan warningTimeout, string processTitle)
{
if (!process.WaitForExit((int)warningTimeout.TotalMilliseconds))
{
Utils.Status($"{processTitle} - Taking a long time to exit.", alsoLog: true);
process.WaitForExit();
Utils.Status($"{processTitle} - Exited after a long period.", alsoLog: true);
}
}
private static Stream CreateSignature(byte[] oldData)
{
Utils.Status("Creating Patch Signature");
using var oldDataStream = new MemoryStream(oldData);
var sigStream = new MemoryStream();
var signatureBuilder = new SignatureBuilder();
signatureBuilder.Build(oldDataStream, new SignatureWriter(sigStream));
sigStream.Position = 0;
return(sigStream);
}
public void ReportProgress(string operation, long currentPosition, long total)
{
if (DateTime.Now - _lastUpdate < _updateInterval)
{
return;
}
_lastUpdate = DateTime.Now;
if (currentPosition >= total || total < 1 || currentPosition < 0)
{
return;
}
Utils.Status(operation, new Percent(total, currentPosition));
}
public static async Task CompactFolder(this AbsolutePath folder, WorkQueue queue, Algorithm algorithm)
{
var driveInfo = folder.DriveInfo().DiskSpaceInfo;
var clusterSize = driveInfo.SectorsPerCluster * driveInfo.BytesPerSector;
await folder
.EnumerateFiles(true)
.Where(f => f.Size > clusterSize)
.PMap(queue, async path =>
{
Utils.Status($"Compacting {path.FileName}");
await path.Compact(algorithm);
});
}
private void UpdateStatus()
{
if (_inner.Length == 0)
{
return;
}
if (_queue != null)
{
_queue.Report(_message, Percent.FactoryPutInRange(_inner.Position, _inner.Length));
}
else
{
Utils.Status(_message, Percent.FactoryPutInRange(_inner.Position, _inner.Length));
}
}
private void UpdateStatus()
{
if (_inner.Length == 0)
{
return;
}
if (_queue != null)
{
_queue.Report(_message, (int)(_inner.Position * 100 / _inner.Length));
}
else
{
Utils.Status(_message, (int)(_inner.Position * 100 / _inner.Length));
}
}
private static async Task ExtractAllWith7Zip(AbsolutePath source, AbsolutePath dest)
{
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper
{
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] { "x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off" }
};
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var(_, line) = p;
if (line == null)
{
return;
}
if (line.Length <= 4 || line[3] != '%')
{
return;
}
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.Error(new _7zipReturnError(exitCode, source, dest, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
}
private static async Task <ExtractedFiles> ExtractAllExe(AbsolutePath source)
{
var isArchive = await TestWith7z(source);
if (isArchive)
{
return(await ExtractAllWith7Zip(source, null));
}
var dest = await TempFolder.Create();
Utils.Log($"Extracting {(string)source.FileName}");
var process = new ProcessHelper
{
Path = @"Extractors\innounp.exe".RelativeTo(AbsolutePath.EntryPoint),
Arguments = new object[] { "-x", "-y", "-b", $"-d\"{dest.Dir}\"", source }
};
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var(_, line) = p;
if (line == null)
{
return;
}
if (line.Length <= 4 || line[3] != '%')
{
return;
}
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
await process.Start();
return(new ExtractedFiles(dest));
}
private void UpdateStatus()
{
if (DateTime.Now - _lastUpdate < TimeSpan.FromMilliseconds(500))
{
return;
}
_lastUpdate = DateTime.Now;
if (_inner.Length == 0)
{
return;
}
if (_queue != null)
{
_queue.Report(_message, Percent.FactoryPutInRange(_inner.Position, _inner.Length));
}
else
{
Utils.Status(_message, Percent.FactoryPutInRange(_inner.Position, _inner.Length));
}
}
public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output)
{
var dataA = a.xxHash();
var dataB = b.xxHash();
var key = PatchKey(dataA, dataB);
var found = _patchCache !.Get(key);
if (found != null)
{
await output.WriteAsync(found);
return;
}
await using var patch = new MemoryStream();
Utils.Status("Creating Patch");
OctoDiff.Create(a, b, patch);
_patchCache.Put(key, patch.ToArray());
patch.Position = 0;
await patch.CopyToAsync(output);
}
private static async Task <Dictionary <RelativePath, T> > GatheringExtractWith7Zip <T>(WorkQueue queue, IStreamFactory sf, Definitions.FileType sig, Predicate <RelativePath> shouldExtract, Func <RelativePath, IExtractedFile, ValueTask <T> > mapfn,
AbsolutePath tempPath, HashSet <RelativePath> onlyFiles)
{
TempFile tmpFile = null;
var dest = tempPath.Combine(Guid.NewGuid().ToString());
dest.CreateDirectory();
TempFile spoolFile = null;
AbsolutePath source;
try
{
if (sf.Name is AbsolutePath abs)
{
source = abs;
}
else
{
spoolFile = new TempFile(tempPath.Combine(Guid.NewGuid().ToString())
.WithExtension(source.Extension));
await using var s = await sf.GetStream();
await spoolFile.Path.WriteAllAsync(s);
source = spoolFile.Path;
}
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}",
$"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper {
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
};
if (onlyFiles != null)
{
//It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy
IEnumerable <string> AllVariants(string input)
{
yield return($"\"{input}\"");
yield return($"\"\\{input}\"");
}
tmpFile = new TempFile();
await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray());
process.Arguments = new object[]
{
"x", "-bsp1", "-y", $"-o\"{dest}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
};
}
else
{
process.Arguments = new object[] { "x", "-bsp1", "-y", $"-o\"{dest}\"", source, "-mmt=off" };
}
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var(_, line) = p;
if (line == null)
{
return;
}
if (line.Length <= 4 || line[3] != '%')
{
return;
}
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}",
Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.ErrorThrow(new _7zipReturnError(exitCode, source, dest, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
var results = await dest.EnumerateFiles()
.PMap(queue, async f =>
{
var path = f.RelativeTo(dest);
if (!shouldExtract(path))
{
return(((RelativePath, T)) default);
public void SetProgress(long inSize, long outSize)
{
Utils.Status("Extracting OMOD", Percent.FactoryPutInRange(inSize, _total));
}
private static async Task <ExtractedFiles> ExtractAllWith7Zip(AbsolutePath source, IEnumerable <RelativePath> onlyFiles)
{
TempFile tmpFile = null;
var dest = await TempFolder.Create();
Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}", $"The contents of {(string)source.FileName} are being extracted to {(string)source.FileName} using 7zip.exe"));
var process = new ProcessHelper
{
Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint),
};
if (onlyFiles != null)
{
//It's stupid that we have to do this, but 7zip's file pattern matching isn't very fuzzy
IEnumerable <string> AllVariants(string input)
{
yield return($"\"{input}\"");
yield return($"\"\\{input}\"");
}
tmpFile = new TempFile();
await tmpFile.Path.WriteAllLinesAsync(onlyFiles.SelectMany(f => AllVariants((string)f)).ToArray());
process.Arguments = new object[]
{
"x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, $"@\"{tmpFile.Path}\"", "-mmt=off"
};
}
else
{
process.Arguments = new object[] { "x", "-bsp1", "-y", $"-o\"{dest.Dir}\"", source, "-mmt=off" };
}
var result = process.Output.Where(d => d.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(p =>
{
var(_, line) = p;
if (line == null)
{
return;
}
if (line.Length <= 4 || line[3] != '%')
{
return;
}
int.TryParse(line.Substring(0, 3), out var percentInt);
Utils.Status($"Extracting {(string)source.FileName} - {line.Trim()}", Percent.FactoryPutInRange(percentInt / 100d));
});
var exitCode = await process.Start();
if (exitCode != 0)
{
Utils.Error(new _7zipReturnError(exitCode, source, dest.Dir, ""));
}
else
{
Utils.Status($"Extracting {source.FileName} - done", Percent.One, alsoLog: true);
}
tmpFile?.Dispose();
return(new ExtractedFiles(dest));
}
private static void ExtractAllWithInno(string source, string dest)
{
Utils.Log($"Extracting {Path.GetFileName(source)}");
var info = new ProcessStartInfo
{
FileName = @"Extractors\innounp.exe",
Arguments = $"-x -y -b -d\"{dest}\" \"{source}\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var p = new Process {
StartInfo = info
};
p.Start();
ChildProcessTracker.AddProcess(p);
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception e)
{
Utils.Error(e, "Error while setting process priority level for innounp.exe");
}
var name = Path.GetFileName(source);
try
{
while (!p.HasExited)
{
var line = p.StandardOutput.ReadLine();
if (line == null)
{
break;
}
if (line.Length <= 4 || line[3] != '%')
{
continue;
}
int.TryParse(line.Substring(0, 3), out var percent);
Utils.Status($"Extracting {name} - {line.Trim()}", percent);
}
}
catch (Exception e)
{
Utils.Error(e, "Error while reading StandardOutput for innounp.exe");
}
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Extracting {name}");
if (p.ExitCode == 0)
{
return;
}
Utils.Log(p.StandardOutput.ReadToEnd());
Utils.Log($"Extraction error extracting {source}");
}
/// <summary>
/// Returns true if the given extension type can be extracted
/// </summary>
/// <param name="v"></param>
/// <returns></returns>
public static bool CanExtract(string v)
{
var ext = Path.GetExtension(v.ToLower());
if (ext != ".exe" && !Consts.TestArchivesBeforeExtraction.Contains(ext))
{
return(Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext));
}
if (ext == ".exe")
{
var info = new ProcessStartInfo
{
FileName = @"Extractors\innounp.exe",
Arguments = $"-t \"{v}\" ",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var p = new Process {
StartInfo = info
};
p.Start();
ChildProcessTracker.AddProcess(p);
var name = Path.GetFileName(v);
while (!p.HasExited)
{
var line = p.StandardOutput.ReadLine();
if (line == null)
{
break;
}
if (line[0] != '#')
{
continue;
}
Utils.Status($"Testing {name} - {line.Trim()}");
}
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Testing {name}");
return(p.ExitCode == 0);
}
var testInfo = new ProcessStartInfo
{
FileName = @"Extractors\7z.exe",
Arguments = $"t \"{v}\"",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var testP = new Process {
StartInfo = testInfo
};
testP.Start();
ChildProcessTracker.AddProcess(testP);
try
{
testP.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
}
try
{
while (!testP.HasExited)
{
var line = testP.StandardOutput.ReadLine();
if (line == null)
{
break;
}
}
} catch (Exception) {}
testP.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Can Extract Check {v}");
return(testP.ExitCode == 0);
}
private static void ExtractAllWith7Zip(string source, string dest)
{
Utils.Log(new GenericInfo($"Extracting {Path.GetFileName(source)}", $"The contents of {source} are being extracted to {dest} using 7zip.exe"));
var info = new ProcessStartInfo
{
FileName = @"Extractors\7z.exe",
Arguments = $"x -bsp1 -y -o\"{dest}\" \"{source}\" -mmt=off",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var p = new Process {
StartInfo = info
};
p.Start();
ChildProcessTracker.AddProcess(p);
try
{
p.PriorityClass = ProcessPriorityClass.BelowNormal;
}
catch (Exception)
{
}
var name = Path.GetFileName(source);
try
{
while (!p.HasExited)
{
var line = p.StandardOutput.ReadLine();
if (line == null)
{
break;
}
if (line.Length <= 4 || line[3] != '%')
{
continue;
}
int.TryParse(line.Substring(0, 3), out var percent);
Utils.Status($"Extracting {name} - {line.Trim()}", percent);
}
}
catch (Exception)
{
}
p.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Extracting {name}");
if (p.ExitCode == 0)
{
Utils.Status($"Extracting {name} - 100%", 100, alsoLog: true);
return;
}
Utils.Error(new _7zipReturnError(p.ExitCode, source, dest, p.StandardOutput.ReadToEnd()));
}
public static async Task <long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash,
Stream?patchOutStream = null)
{
if (patchOutStream == null)
{
await using var rcmd = new SQLiteCommand(_conn);
rcmd.CommandText = "SELECT PatchSize FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
rcmd.Parameters.AddWithValue("@fromHash", (long)srcHash);
rcmd.Parameters.AddWithValue("@toHash", (long)destHash);
await using var rdr = await rcmd.ExecuteReaderAsync();
while (await rdr.ReadAsync())
{
return(rdr.GetInt64(0));
}
}
else
{
if (TryGetPatch(srcHash, destHash, out var entry))
{
await patchOutStream !.WriteAsync(await entry.GetData());
return(entry.PatchSize);
}
}
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch)
VALUES (@fromHash, @toHash, @patchSize, @patch)";
cmd.Parameters.AddWithValue("@fromHash", (long)srcHash);
cmd.Parameters.AddWithValue("@toHash", (long)destHash);
Utils.Status("Creating Patch");
await using var sigStream = new MemoryStream();
await using var patchStream = new MemoryStream();
OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
cmd.Parameters.AddWithValue("@patchSize", patchStream.Length);
cmd.Parameters.AddWithValue("@patch", patchStream.ToArray());
try
{
await cmd.ExecuteNonQueryAsync();
}
catch (SQLiteException ex)
{
if (!ex.Message.StartsWith("constraint failed"))
{
throw;
}
}
if (patchOutStream == null)
{
return(patchStream.Position);
}
patchStream.Position = 0;
await patchStream.CopyToAsync(patchOutStream);
return(patchStream.Position);
}
public ProgressReporter()
{
_updateInterval = TimeSpan.FromMilliseconds(100);
_report = (s, percent) => Utils.Status(s, percent);
}
