public void OpusArchiveEncoderTest()
{
ExtendedArchiveWriter writer = new ExtendedArchiveWriter("opusencoder");
writer.Files.Add(new Subfile(new FileSource(WavTestFile), "audio.wav", "arc"));
writer.Write("opusencodertest.ppx");
ExtendedArchive arc = new ExtendedArchive("opusencodertest.ppx");
var subfile = arc.Files.First();
Assert.AreEqual(ArchiveFileType.OpusAudio, subfile.Type);
Assert.IsTrue(subfile.Name == "audio.opus", $"Internal name did not switch to \"audio.opus\". Actual: {subfile.Name}");
Assert.IsTrue(subfile.EmulatedName == "audio.wav", $"Emulated name did stay as \"audio.wav\". Actual: {subfile.EmulatedName}");
Assert.IsTrue(subfile.ArchiveName == "arc", $"Archive name did not stay as \"arc\". Actual: {subfile.ArchiveName}");
Assert.IsTrue(subfile.EmulatedArchiveName == "arc.pp", $"Emulated archive name did not change to \"arc.pp\". Actual: {subfile.EmulatedArchiveName}");
using (OpusEncoder decoder = new OpusEncoder(subfile.GetRawStream()))
{
Stream decoded = decoder.Decode();
//Ensure it can be read
using (WaveReader wavreader = new WaveReader(decoded)) { }
}
File.Delete("opusencodertest.ppx");
}
public static void TeardownPPX(ExtendedArchive archive)
{
if (File.Exists(archive.Filename))
{
File.Delete(archive.Filename);
}
}
public static void Initialize(TestContext context)
{
FileStream arc = new FileStream("test.ppx", FileMode.Create);
var writer = new ExtendedArchiveWriter("test", true);
using (var mem = new MemoryStream(TestData))
TestHash = PPeX.Utility.GetMd5(mem);
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test1",
"t",
ArchiveFileType.Raw));
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test2",
"t",
ArchiveFileType.Raw));
writer.Write(arc);
arc.Close();
TestArchive = new ExtendedArchive("test.ppx");
cache = new CompressedCache(new[] { TestArchive });
}
private void btnDecompress_Click(object sender, EventArgs e)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
var archive = new ExtendedArchive(textBox1.Text);
string dir = Path.Combine(Path.GetDirectoryName(textBox1.Text), Path.GetFileNameWithoutExtension(textBox1.Text));
Directory.CreateDirectory(dir);
sw.Stop();
MessageBox.Show(sw.Elapsed.TotalMilliseconds.ToString());
sw.Restart();
foreach (var file in archive.ArchiveFiles)
{
string filename = dir + "\\" + file.Name;
using (FileStream fs = new FileStream(filename, FileMode.Create))
using (Stream source = file.GetStream())
{
source.CopyTo(fs);
}
}
sw.Stop();
MessageBox.Show(sw.Elapsed.TotalMilliseconds.ToString());
}
private void lsvMD_DragDrop(object sender, DragEventArgs e)
{
string[] files = (string[])e.Data.GetData(DataFormats.FileDrop);
foreach (string file in files)
{
int check;
ExtendedArchive ex = new ExtendedArchive(file);
foreach (var item in ex.ArchiveFiles)
{
string md5 = ByteArrayToString(item.Md5);
if (!MD5counter.TryGetValue(md5, out check))
{
if (!item.Flags.HasFlag(ArchiveFileFlags.Duplicate))
{
MD5counter[md5] = 1;
filesizes[md5] = (int)item.Size;
}
}
else
{
MD5counter[md5] = 1 + check;
}
}
}
Recalchash();
}
public static void Initialize(TestContext context)
{
FileStream arc = new FileStream("test.ppx", FileMode.Create);
var writer = new ExtendedArchiveWriter("test", true);
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test1",
"t",
ArchiveFileType.Raw));
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test2",
"t",
ArchiveFileType.Raw));
writer.Files.Add(new Subfile(
new MemorySource(TestData2),
"test3",
"t",
ArchiveFileType.Raw));
writer.Write(arc);
arc.Close();
TestArchive = new ExtendedArchive("test.ppx");
}
protected void AssertCRC(ExtendedArchive arc)
{
foreach (var chunk in arc.Chunks)
{
Assert.IsTrue(chunk.VerifyChecksum());
}
}
public void DecompressionSpeedTest()
{
Stopwatch sw = new Stopwatch();
sw.Start();
var archive = new ExtendedArchive(@"I:\AA2\Artificial Academy 2\Artificial Academy 2\data\05\jg2p06_00_002png.ppx");
sw.Stop();
Trace.WriteLine("Setup: " + Math.Round(sw.Elapsed.TotalMilliseconds, 4) + "ms");
sw.Restart();
TimeSpan old = sw.Elapsed;
foreach (var file in archive.ArchiveFiles)
{
using (MemoryStream mem = new MemoryStream())
using (Stream source = file.GetStream())
{
source.CopyTo(mem);
sw.Stop();
double mbs = (file.Size / (1024 * 1024 * ((sw.Elapsed.TotalMilliseconds - old.TotalMilliseconds) / 1000)));
Trace.WriteLine(file.Name + ": " + Math.Round(mbs, 1) + "mb/s");
old = sw.Elapsed;
sw.Start();
}
}
sw.Stop();
Trace.WriteLine("Total: " + sw.Elapsed.TotalMilliseconds + "ms");
}
public void Xx4ArchiveEncoderTest()
{
ExtendedArchiveWriter writer = new ExtendedArchiveWriter("xx4encoder");
writer.Files.Add(new Subfile(new FileSource(XxTestFile), "mesh.xx", "arc"));
writer.Write("xx4encodertest.ppx");
ExtendedArchive arc = new ExtendedArchive("xx4encodertest.ppx");
var subfile = arc.Files.First();
Assert.AreEqual(ArchiveFileType.Xx4Mesh, subfile.Type);
Assert.IsTrue(subfile.Name == "mesh.xx4", $"Internal name did not switch to \"mesh.xx4\". Actual: {subfile.Name}");
Assert.IsTrue(subfile.EmulatedName == "mesh.xx", $"Emulated name did stay as \"mesh.xx\". Actual: {subfile.EmulatedName}");
Assert.IsTrue(subfile.ArchiveName == "arc", $"Archive name did not stay as \"arc\". Actual: {subfile.ArchiveName}");
Assert.IsTrue(subfile.EmulatedArchiveName == "arc.pp", $"Emulated archive name did not change to \"arc.pp\". Actual: {subfile.EmulatedArchiveName}");
using (Xx4Encoder decoder = new Xx4Encoder(subfile.GetRawStream(), arc.TextureBank))
{
Stream decoded = decoder.Decode();
}
File.Delete("xx4encodertest.ppx");
}
public void Open()
{
string path = ShowOpenFileDialog("Extended PP archive|*.ppx").FirstOrDefault();
if (string.IsNullOrEmpty(path))
{
return;
}
CloseFile();
OpenFile();
currentlyOpenedFile = Path.GetFileName(path);
IsModified = false;
ExtendedArchive arc = new ExtendedArchive(path);
List <TreeNode> parents = new List <TreeNode>();
trvFiles.BeginUpdate();
foreach (var file in arc.RawFiles)
{
TreeNode parent = parents.FirstOrDefault(x => x.Text == file.ArchiveName);
if (parent == null)
{
parent = trvFiles.Nodes.Add(file.ArchiveName);
parents.Add(parent);
}
TreeNode node = parent.Nodes.Add(file.Name);
node.Tag = new SubfileHolder(file, file.Name);
SetAutoIcon(node);
}
trvFiles.EndUpdate();
foreach (var chunk in arc.Chunks)
{
var item = lsvChunks.Items.Add(chunk.ID.ToString());
item.SubItems.Add(PPeX.Utility.GetBytesReadable((long)chunk.UncompressedLength));
item.SubItems.Add(PPeX.Utility.GetBytesReadable((long)chunk.CompressedLength));
item.SubItems.Add(chunk.Files.Count.ToString());
item.SubItems.Add(chunk.Offset.ToString());
item.SubItems.Add(chunk.Compression.ToString());
if (chunk.Files.Any(x => x.ArchiveName.Contains("TextureBank")))
{
item.BackColor = Color.LightSalmon;
}
}
lblFileCount.Text = "Total files: " + arc.Chunks.Sum(x => x.Files.Count) + " - " + PPeX.Utility.GetBytesReadable(arc.Chunks.Sum(x => (long)x.UncompressedLength));
}
private void btnRetype_Click(object sender, EventArgs e)
{
foreach (ListViewItem item in lsvFiles.Items)
{
ExtendedArchive ex = new ExtendedArchive(item.Text);
ExtendedArchiveWriter wr = new ExtendedArchiveWriter("B:\\" + Path.GetFileName(item.Text), (ex.Title + ".pp").Replace(".pp.pp", ".pp"));
foreach (var s in ex.ArchiveFiles)
{
//wr.Files.Add(new ArchiveFile(s, s.Name));
}
wr.Write();
}
}
public void AppendMetadataTest()
{
var ppx = TestCommon.GeneratePPX();
ExtendedArchiveAppender appender = new ExtendedArchiveAppender(ppx);
appender.Name = "test-appended";
var file1 = new Subfile(
new MemorySource(TestData),
"test3-2",
"t",
ArchiveFileType.Raw);
var file2 = new Subfile(
new MemorySource(Encoding.UTF8.GetBytes("short data")),
"test4-2",
"t",
ArchiveFileType.Raw);
var file3 = appender.BaseArchive.RawFiles.Find(x => x.Name == "test00");
appender.FilesToAdd.Add(file1);
appender.FilesToAdd.Add(file2);
appender.FilesToRemove.Add(file3);
appender.Write();
ppx = new ExtendedArchive(ppx.Filename);
Assert.AreEqual("test-appended", ppx.Title);
Assert.AreEqual(33, ppx.Files.Count);
Assert.IsTrue(ppx.Files.Any(x => x.Name == file1.Name));
Assert.IsTrue(ppx.Files.Any(x => x.Name == file2.Name));
Assert.IsFalse(ppx.Files.Any(x => x.Name == file3.Name));
AssertCRC(ppx);
TestCommon.TeardownPPX(ppx);
}
public ppxEditor(ExtendedArchive archive)
{
if (ExternalTools == null && Gui.Config != null)
{
ExternalTools = new Dictionary <string, List <ExternalTool> >();
foreach (SettingsPropertyValue value in Gui.Config.PropertyValues)
{
if (value.PropertyValue.GetType() == typeof(ExternalTool))
{
ExternalTool tool = (ExternalTool)value.PropertyValue;
tool = RegisterExternalTool(tool, false);
string toolVar = Gui.Scripting.GetNextVariable("externalTool");
Gui.Scripting.Variables.Add(toolVar, tool);
}
}
}
Appender = new ExtendedArchiveAppender(archive);
}
public void Open()
{
CommonOpenFileDialog dialog = new CommonOpenFileDialog()
{
Multiselect = false
};
dialog.Filters.Add(new CommonFileDialogFilter("Extended PP archive", "*.ppx"));
var result = dialog.ShowDialog();
if (result != CommonFileDialogResult.Ok)
{
return;
}
CloseFile();
OpenFile();
currentlyOpenedFile = Path.GetFileName(dialog.FileName);
IsModified = false;
ExtendedArchive arc = new ExtendedArchive(dialog.FileName);
foreach (var file in arc.ArchiveFiles)
{
TreeNode parent = trvFiles.Nodes.Cast <TreeNode>().FirstOrDefault(x => x.Text == file.ArchiveName);
if (parent == null)
{
parent = trvFiles.Nodes.Add(file.ArchiveName);
}
TreeNode node = parent.Nodes.Add(file.Name);
node.Tag = new SubfileHolder(file, file.Name);
SetAutoIcon(node);
}
}
public formVerify(ExtendedArchive arc)
{
InitializeComponent();
this.arc = arc;
}
static async Task DecompressArg(string[] args)
{
Regex regex = new Regex(".+");
int argcounter = 1;
string decode = "off";
string currentArg;
while ((currentArg = args[argcounter++]).StartsWith("-"))
{
currentArg = currentArg.ToLower();
if (currentArg == "-decode")
{
decode = args[argcounter++];
}
else if (currentArg == "-regex")
{
regex = new Regex(args[argcounter++]);
}
}
decode = decode.ToLower();
ExtendedArchive archive = new ExtendedArchive(currentArg);
DirectoryInfo outputDirectory = new DirectoryInfo(args[argcounter++]);
if (!outputDirectory.Exists)
{
outputDirectory.Create();
}
var chunks = archive.RawFiles
.Where(x => regex.IsMatch($"{x.ArchiveName}/{x.Name}"))
.GroupBy(x => x.ChunkID);
foreach (var chunkListing in chunks)
{
var chunk = chunkListing.First().Chunk;
using var decompressionBuffer = MemoryPool <byte> .Shared.Rent((int) chunk.UncompressedLength);
chunk.CopyToMemory(decompressionBuffer.Memory);
List <Task> tasks = new List <Task>();
foreach (var file in chunkListing)
{
string arcname = file.ArchiveName.Replace(".pp", "");
Console.WriteLine($"Exporting {file.ArchiveName}/{file.Name}");
if (!Directory.Exists(Path.Combine(outputDirectory.FullName, arcname)))
{
Directory.CreateDirectory(Path.Combine(outputDirectory.FullName, arcname));
}
tasks.Add(Task.Run(async() =>
{
await using FileStream fs = new FileStream(Path.Combine(outputDirectory.FullName, arcname, file.Name), FileMode.Create);
await fs.WriteAsync(decompressionBuffer.Memory.Slice((int)file.Offset, (int)file.Size));
}));
}
await Task.WhenAll(tasks);
}
//else if (decode == "on" || decode == "full")
//{
// //decoding raw entries
// foreach (var file in archive.Files)
// {
// string fullname = file.ArchiveName + "/" + file.Name;
// string arcname = file.ArchiveName.Replace(".pp", "");
// if (regex.IsMatch(fullname))
// {
// Console.WriteLine("Exporting " + fullname);
// if (!Directory.Exists(Path.Combine(outputDirectory.FullName, arcname)))
// Directory.CreateDirectory(Path.Combine(outputDirectory.FullName, arcname));
// if (decode == "full" && file.Type == ArchiveFileType.OpusAudio)
// {
// string fileName = file.Name.Replace(".opus", ".wav");
// using (FileStream fs = new FileStream(Path.Combine(outputDirectory.FullName, arcname, fileName), FileMode.Create))
// using (Stream stream = file.GetRawStream())
// {
// stream.CopyTo(fs);
// }
// }
// else if (file.Type != ArchiveFileType.OpusAudio)
// {
// string fileName;
// fileName = EncoderFactory.TransformName(file.Name, file.Type);
// using (FileStream fs = new FileStream(Path.Combine(outputDirectory.FullName, arcname, fileName), FileMode.Create))
// using (Stream stream = file.GetRawStream())
// {
// stream.CopyTo(fs);
// }
// }
// else
// {
// using (FileStream fs = new FileStream(Path.Combine(outputDirectory.FullName, arcname, file.Name), FileMode.Create))
// using (Stream stream = (file as ArchiveSubfile).RawSource.GetStream())
// {
// stream.CopyTo(fs);
// }
// }
// }
// }
//}
}
static void VerifyArgs(string[] args)
{
string currentArg;
int argcounter = 1;
bool fullVerify = true;
while ((currentArg = args[argcounter++]).StartsWith("-"))
{
currentArg = currentArg.ToLower();
if (currentArg == "-fullverify")
{
fullVerify = args[argcounter++].ToLower() == "on";
}
else
{
HaltAndCatchFire($"Unknown command: \"{currentArg}\"", 1);
return;
}
}
string filename = args[--argcounter];
if (!File.Exists(filename))
{
HaltAndCatchFire($"Input file does not exist: {filename}", 1);
}
ExtendedArchive arc;
try
{
arc = new ExtendedArchive(filename);
}
catch (PpexException ex)
{
if (ex.ErrorCode == PpexException.PpexErrorCode.FileNotPPXArchive ||
ex.ErrorCode == PpexException.PpexErrorCode.IncorrectVersionNumber)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex.Message);
Console.ResetColor();
return;
}
else
{
throw;
}
}
Console.CursorVisible = false;
Action <int, string> UpdateProgress = (i, x) =>
{
Console.SetCursorPosition(0, Console.CursorTop);
string message = $"Verification {i}% complete [{x}]";
message = message.PadRight(Console.BufferWidth - 1);
Console.Write(message);
};
var orderedChunks = arc.Chunks.OrderBy(x => x.Offset).ToArray();
Console.WriteLine("Archive name: " + arc.Title);
Console.WriteLine("File count: " + arc.Files.Count);
long currentOffset = 0;
long totalOffset = orderedChunks.Sum(x => (long)x.CompressedLength);
HashSet <(Md5Hash, ulong, ulong)> verifiedHashes = new HashSet <(Md5Hash, ulong, ulong)>();
for (int i = 0; i < orderedChunks.Length; i++)
{
ExtendedArchiveChunk chunk = orderedChunks[i];
using var compressedBuffer = MemoryPool <byte> .Shared.Rent((int) chunk.CompressedLength);
var compressedMemory = compressedBuffer.Memory.Slice(0, (int)chunk.CompressedLength);
using (var rawStream = chunk.GetRawStream())
{
rawStream.Read(compressedMemory.Span);
}
if (chunk.CRC32 != CRC32.Compute(compressedMemory.Span))
{
HaltAndCatchFire($"Chunk hash mismatch; id: {chunk.ID}, offset: {chunk.Offset}", 8);
}
if (fullVerify)
{
using var uncompressedBuffer = MemoryPool <byte> .Shared.Rent((int) chunk.UncompressedLength);
var uncompressedMemory = uncompressedBuffer.Memory.Slice(0, (int)chunk.UncompressedLength);
chunk.CopyToMemory(uncompressedMemory);
foreach (var file in chunk.Files)
{
var expectedHash = (file.RawSource.Md5, file.RawSource.Offset, file.RawSource.Size);
if (verifiedHashes.Contains(expectedHash))
{
continue;
}
var actualMd5 =
Utility.GetMd5(uncompressedMemory.Span.Slice((int)file.RawSource.Offset,
(int)file.RawSource.Size));
if ((Md5Hash)file.RawSource.Md5 != (Md5Hash)actualMd5)
{
HaltAndCatchFire($"File md5 mismatch; chunk id: {chunk.ID}, archive: {file.ArchiveName}, file: {file.Name}", 9);
}
verifiedHashes.Add((actualMd5, file.RawSource.Offset, file.RawSource.Size));
}
verifiedHashes.Clear();
}
currentOffset += (long)chunk.CompressedLength;
//int percentage = (int)((float)(100 * i) / (float)orderedChunks.Length);
int percentage = (int)Math.Round((float)(100 * currentOffset) / (float)totalOffset);
UpdateProgress(percentage, $"{i + 1}/{orderedChunks.Length} : {ConvertToReadable(currentOffset)}/{ConvertToReadable(totalOffset)}");
}
Console.WriteLine();
Console.WriteLine($"\"{arc.Title}\" successfully verified.");
Console.CursorVisible = true;
}
static void Main(string[] args)
{
server = new PipeServer("PPEX");
#if DEBUG
LogFiles = true;
#endif
TrimTimer = new Timer(TurboMode ? 10000 : 5000);
if (args.Length > 0 &&
Directory.Exists(args[0]))
{
PPXLocation = args[0];
}
PPXLocation = PPXLocation.Replace("\\\\", "\\");
if (args.Length > 0 &&
args.Any(x => x.ToLower() == "-nowindow"))
{
ShowWindow(GetConsoleWindow(), SW_HIDE);
}
//Attach the handler to the server
server.OnRequest += Server_OnRequest;
server.OnDisconnect += Server_OnDisconnect;
if (!Directory.Exists(PPXLocation))
{
Console.WriteLine("Invalid load directory! (" + PPXLocation + ")");
return;
}
Console.WriteLine("Loading from " + PPXLocation);
List <ExtendedArchive> ArchivesToLoad = new List <ExtendedArchive>();
//Index all .ppx files in the location
foreach (string arc in Directory.EnumerateFiles(PPXLocation, "*.ppx", SearchOption.TopDirectoryOnly)
.OrderBy(x => x))
{
var archive = new ExtendedArchive(arc);
ArchivesToLoad.Add(archive);
}
const long cacheSize = (long)(1.8 * 1024 * 1024 * 1024);
const long trimSize = (long)(cacheSize * 0.9);
Cache = new CompressedCache(ArchivesToLoad, new Progress <string>(Console.WriteLine), cacheSize);
TrimTimer.Elapsed += (s, e) =>
{
Cache.Trim(trimSize);
Console.WriteLine("Cache size:" + Utility.GetBytesReadable(Cache.AllocatedMemorySize));
};
TrimTimer.Start();
Console.WriteLine("Finished loading " + Cache.LoadedArchives.Count + " archive(s)");
if (Preloading)
{
Console.WriteLine("Preloading files...");
foreach (var chunk in Cache.ReferenceMd5Sums.Keys.Where(x => x.File.EndsWith(".lst")).Select(x => Cache.LoadedFileReferences[x]).Distinct())
{
chunk.Allocate();
}
//foreach (var chunk in Cache.LoadedFiles.Where(x => x.Key.Archive.StartsWith("jg2p06")).Select(x => x.Value.Chunk).Distinct())
// chunk.Allocate();
foreach (var chunk in Cache.ReferenceMd5Sums.Keys.Where(x => x.File.EndsWith(".bmp")).Select(x => Cache.LoadedFileReferences[x]).Distinct())
{
chunk.Allocate();
}
foreach (var chunk in Cache.ReferenceMd5Sums.Keys.Where(x => x.File.EndsWith(".tga")).Select(x => Cache.LoadedFileReferences[x]).Distinct())
{
chunk.Allocate();
}
//foreach (var chunk in Cache.LoadedFiles.Where(x => x.Key.Archive.StartsWith("jg2p07")).Select(x => x.Value.Chunk).Distinct())
// chunk.Allocate();
Console.WriteLine("Preloading complete.");
}
Cache.Trim(TrimMethod.GCCompactOnly);
IsLoaded = true;
string line;
//Handle arguments from the user
while (true)
{
line = Console.ReadLine();
string[] arguments = line.Split(' ');
switch (arguments[0])
{
case "exit":
Environment.Exit(0);
return;
case "log":
LogFiles = !LogFiles;
break;
case "turbo":
TurboMode = !TurboMode;
break;
case "size":
Console.WriteLine(Utility.GetBytesReadable(Cache.AllocatedMemorySize));
Console.WriteLine(Cache.LoadedFiles.Count + " files allocated");
break;
case "trim":
Cache.Trim((long)(float.Parse(arguments[1]) * 1024 * 1024));
break;
}
}
}
