public static void Initialize(TestContext context)
{
FileStream arc = new FileStream("test.ppx", FileMode.Create);
var writer = new ExtendedArchiveWriter("test", true);
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test1",
"t",
ArchiveFileType.Raw));
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test2",
"t",
ArchiveFileType.Raw));
writer.Files.Add(new Subfile(
new MemorySource(TestData2),
"test3",
"t",
ArchiveFileType.Raw));
writer.Write(arc);
arc.Close();
TestArchive = new ExtendedArchive("test.ppx");
}
public static void Initialize(TestContext context)
{
FileStream arc = new FileStream("test.ppx", FileMode.Create);
var writer = new ExtendedArchiveWriter("test", true);
using (var mem = new MemoryStream(TestData))
TestHash = PPeX.Utility.GetMd5(mem);
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test1",
"t",
ArchiveFileType.Raw));
writer.Files.Add(new Subfile(
new MemorySource(TestData),
"test2",
"t",
ArchiveFileType.Raw));
writer.Write(arc);
arc.Close();
TestArchive = new ExtendedArchive("test.ppx");
cache = new CompressedCache(new[] { TestArchive });
}
public void OpusArchiveEncoderTest()
{
ExtendedArchiveWriter writer = new ExtendedArchiveWriter("opusencoder");
writer.Files.Add(new Subfile(new FileSource(WavTestFile), "audio.wav", "arc"));
writer.Write("opusencodertest.ppx");
ExtendedArchive arc = new ExtendedArchive("opusencodertest.ppx");
var subfile = arc.Files.First();
Assert.AreEqual(ArchiveFileType.OpusAudio, subfile.Type);
Assert.IsTrue(subfile.Name == "audio.opus", $"Internal name did not switch to \"audio.opus\". Actual: {subfile.Name}");
Assert.IsTrue(subfile.EmulatedName == "audio.wav", $"Emulated name did stay as \"audio.wav\". Actual: {subfile.EmulatedName}");
Assert.IsTrue(subfile.ArchiveName == "arc", $"Archive name did not stay as \"arc\". Actual: {subfile.ArchiveName}");
Assert.IsTrue(subfile.EmulatedArchiveName == "arc.pp", $"Emulated archive name did not change to \"arc.pp\". Actual: {subfile.EmulatedArchiveName}");
using (OpusEncoder decoder = new OpusEncoder(subfile.GetRawStream()))
{
Stream decoded = decoder.Decode();
//Ensure it can be read
using (WaveReader wavreader = new WaveReader(decoded)) { }
}
File.Delete("opusencodertest.ppx");
}
public void Xx4ArchiveEncoderTest()
{
ExtendedArchiveWriter writer = new ExtendedArchiveWriter("xx4encoder");
writer.Files.Add(new Subfile(new FileSource(XxTestFile), "mesh.xx", "arc"));
writer.Write("xx4encodertest.ppx");
ExtendedArchive arc = new ExtendedArchive("xx4encodertest.ppx");
var subfile = arc.Files.First();
Assert.AreEqual(ArchiveFileType.Xx4Mesh, subfile.Type);
Assert.IsTrue(subfile.Name == "mesh.xx4", $"Internal name did not switch to \"mesh.xx4\". Actual: {subfile.Name}");
Assert.IsTrue(subfile.EmulatedName == "mesh.xx", $"Emulated name did stay as \"mesh.xx\". Actual: {subfile.EmulatedName}");
Assert.IsTrue(subfile.ArchiveName == "arc", $"Archive name did not stay as \"arc\". Actual: {subfile.ArchiveName}");
Assert.IsTrue(subfile.EmulatedArchiveName == "arc.pp", $"Emulated archive name did not change to \"arc.pp\". Actual: {subfile.EmulatedArchiveName}");
using (Xx4Encoder decoder = new Xx4Encoder(subfile.GetRawStream(), arc.TextureBank))
{
Stream decoded = decoder.Decode();
}
File.Delete("xx4encodertest.ppx");
}
private void btnRetype_Click(object sender, EventArgs e)
{
foreach (ListViewItem item in lsvFiles.Items)
{
ExtendedArchive ex = new ExtendedArchive(item.Text);
ExtendedArchiveWriter wr = new ExtendedArchiveWriter("B:\\" + Path.GetFileName(item.Text), (ex.Title + ".pp").Replace(".pp.pp", ".pp"));
foreach (var s in ex.ArchiveFiles)
{
//wr.Files.Add(new ArchiveFile(s, s.Name));
}
wr.Write();
}
}
public static ExtendedArchive GeneratePPX()
{
//generate random data
Random random = new Random();
TestData = new byte[filecount][];
for (int i = 0; i < filecount; i++)
{
TestData[i] = new byte[512];
random.NextBytes(TestData[i]);
}
string name = "test_ppx" + random.Next(0, 255).ToString();
//create archive
var writer = new ExtendedArchiveWriter(name, true);
writer.ChunkSizeLimit = 1024;
//add files + duplicates
for (int i = 0; i < 8; i++)
{
for (int d = 0; d < 4; d++)
{
ISubfile file = new Subfile(new MemorySource(TestData[i]), $"test{i}{d}", "test", ArchiveFileType.Raw);
writer.Files.Add(file);
}
}
using (FileStream arc = new FileStream($"{name}.ppx", FileMode.Create))
writer.Write(arc);
return(new ExtendedArchive($"{name}.ppx"));
}
public void Save()
{
if (txtArchiveName.Text == "" || txtSaveLocation.Text == "")
{
return;
}
btnSave.Enabled = false;
ExtendedArchiveWriter writer = new ExtendedArchiveWriter(txtArchiveName.Text);
writer.DefaultCompression = (ArchiveChunkCompression)cmbCompression.SelectedIndex;
writer.ChunkSizeLimit = (ulong)numChunkSize.Value * 1024 * 1024;
writer.Threads = (int)numThreads.Value;
Core.Settings.Xx2Precision = (int)numXx2Precision.Value;
Core.Settings.OpusMusicBitrate = (int)(numMusicBitrate.Value * 1000);
Core.Settings.OpusVoiceBitrate = (int)(numVoiceBitrate.Value * 1000);
IProgress <string> progressStatus = new Progress <string>(x =>
{
txtSaveProg.AppendText(x);
});
IProgress <int> progressPercentage = new Progress <int>(x =>
{
prgSaveProgress.Value = x;
});
//attempt loading md5 cache
Core.Settings.UseMd5Cache = chkMd5Cache.Checked;
if (Core.Settings.UseMd5Cache && File.Exists("HashCache.md5.zs"))
{
progressStatus.Report("Loading MD5 cache...\r\n");
progressPercentage.Report(0);
using (var decom = new ZstdDecompressor())
{
string rawCache = Encoding.ASCII.GetString(decom.Unwrap(File.ReadAllBytes("HashCache.md5.zs")));
Core.Settings.Md5Cache = rawCache.Split('\n').Select(x => CachedMd5.FromString(x)).ToDictionary(x => x.Filename);
}
}
Task.Run(() =>
{
FileStream arc = new FileStream(txtSaveLocation.Text, FileMode.Create);
try
{
progressStatus.Report("Performing first pass...\r\n");
progressPercentage.Report(0);
var allNodes = trvFiles.Nodes
.Cast <TreeNode>()
.SelectMany(GetNodeBranch);
int i = 1;
int total = allNodes.Count();
foreach (TreeNode node in allNodes)
{
if (node.Tag == null)
{
continue;
}
var holder = node.Tag as SubfileHolder;
ISubfile subfile = new PPeX.Subfile(
holder.Source,
node.Text,
node.Parent.Text);
writer.Files.Add(subfile);
i++;
if (i % 20 == 0)
{
progressPercentage.Report(100 * i / total);
}
}
writer.Write(arc, progressStatus, progressPercentage);
btnSave.DynamicInvoke(() => btnSave.Enabled = true);
}
#if !DEBUG
catch (Exception ex)
{
progressStatus.Report("ERROR: " + ex.Message + "\n");
progressPercentage.Report(0);
}
#endif
finally
{
arc.Close();
//write hash cache
if (Core.Settings.UseMd5Cache)
{
using (var comp = new ZstdCompressor(new ZstdCompressionOptions(3)))
{
var strings = Core.Settings.Md5Cache.Values.Select(x => x.ToWritableString());
if (strings.Count() > 0)
{
string rawCache = strings.Aggregate((x, y) => x + '\n' + y);
File.WriteAllBytes("HashCache.md5.zs", comp.Wrap(Encoding.ASCII.GetBytes(rawCache)));
}
}
}
this.Invoke(new MethodInvoker(() =>
{
currentlyOpenedFile = Path.GetFileName(txtSaveLocation.Text);
IsModified = false;
btnSave.Enabled = true;
}));
}
});
}
static async Task CompressArg(string[] args)
{
Regex regex = new Regex(".+");
Regex unencodedRegex = new Regex("a^");
int argcounter = 1;
int chunksize = ConvertFromReadable("16M");
int threads = 2;
string name = Path.GetFileNameWithoutExtension(args.Last());
string compression = "zstd";
string currentArg;
Dictionary <ArchiveFileType, ArchiveFileType> encodingTransforms = Core.Settings.DefaultEncodingConversions;
if (Environment.OSVersion.Platform == PlatformID.Unix)
{
Core.Settings.OpusFrameSize = 0.060;
}
while ((currentArg = args[argcounter++]).StartsWith("-"))
{
currentArg = currentArg.ToLower();
if (currentArg == "-chunksize")
{
chunksize = ConvertFromReadable(args[argcounter++]);
}
else if (currentArg == "-opus-music")
{
Core.Settings.OpusMusicBitrate = ConvertFromReadable(args[argcounter++]);
}
else if (currentArg == "-opus-voice")
{
Core.Settings.OpusVoiceBitrate = ConvertFromReadable(args[argcounter++]);
}
else if (currentArg == "-xx2-precision")
{
Core.Settings.Xx2Precision = int.Parse(args[argcounter++]);
Core.Settings.Xx2IsUsingQuality = false;
}
else if (currentArg == "-xx2-quality")
{
Core.Settings.Xx2Quality = float.Parse(args[argcounter++]);
Core.Settings.Xx2IsUsingQuality = false;
}
else if (currentArg == "-threads")
{
threads = int.Parse(args[argcounter++]);
}
else if (currentArg == "-compression")
{
compression = args[argcounter++].ToLower();
}
else if (currentArg == "-zstd-level")
{
Core.Settings.ZstdCompressionLevel = int.Parse(args[argcounter++]);
}
else if (currentArg == "-name")
{
name = args[argcounter++];
}
else if (currentArg == "-regex")
{
regex = new Regex(args[argcounter++]);
}
else if (currentArg == "-no-encode")
{
unencodedRegex = new Regex(args[argcounter++]);
}
else if (currentArg == "-convert")
{
string[] codecs = args[argcounter++].Split(':');
if (codecs.Length != 2)
{
HaltAndCatchFire($"Invalid convert argument: {args[argcounter]}", 100);
}
ArchiveFileType input;
if (!Enum.TryParse(codecs[0], out input))
{
HaltAndCatchFire($"Invalid input codec for conversion: {codecs[0]}", 101);
}
if (string.IsNullOrWhiteSpace(codecs[1]))
{
if (encodingTransforms.ContainsKey(input))
{
encodingTransforms.Remove(input);
}
}
else
{
ArchiveFileType output;
if (!Enum.TryParse(codecs[1], out output))
{
HaltAndCatchFire($"Invalid output codec for conversion: {codecs[1]}", 102);
}
encodingTransforms[input] = output;
}
}
else
{
HaltAndCatchFire($"Unknown command: \"{currentArg}\"", 1);
return;
}
}
compression = compression.ToLower();
string filename = args.Last();
ExtendedArchiveWriter writer = new ExtendedArchiveWriter(name);
writer.ChunkSizeLimit = (ulong)chunksize;
writer.Threads = threads;
writer.EncodingConversions.Clear();
foreach (var kv in encodingTransforms)
{
writer.EncodingConversions[kv.Key] = kv.Value;
}
if (compression == "zstd")
{
writer.DefaultCompression = ArchiveChunkCompression.Zstandard;
}
else if (compression == "lz4")
{
writer.DefaultCompression = ArchiveChunkCompression.LZ4;
}
else if (compression == "uncompressed")
{
writer.DefaultCompression = ArchiveChunkCompression.Uncompressed;
}
foreach (string path in args.Skip(argcounter - 1).Take(args.Length - argcounter))
{
if (path.EndsWith(".pp") && File.Exists(path))
{
Console.WriteLine("Importing " + Path.GetFileName(path));
ImportPP(path, writer.Files, regex, unencodedRegex);
continue;
}
string parentPath = Path.GetDirectoryName(path);
string localPath = Path.GetFileName(path);
foreach (string filepath in Directory.EnumerateFiles(parentPath, localPath, SearchOption.TopDirectoryOnly))
{
if (Path.GetFileName(filepath) == "base.pp")
{
continue;
}
if (filepath.EndsWith(".pp"))
{
//.pp file
Console.WriteLine("Importing " + Path.GetFileName(filepath));
ImportPP(filepath, writer.Files, regex, unencodedRegex);
}
}
foreach (string dirpath in Directory.EnumerateDirectories(parentPath, localPath, SearchOption.TopDirectoryOnly))
{
name = Path.GetFileNameWithoutExtension(dirpath) + ".pp";
Console.WriteLine("Importing \"" + dirpath + "\" as \"" + name + "\"");
int imported = 0;
var files = Directory.EnumerateFiles(dirpath, "*.*", SearchOption.TopDirectoryOnly).ToArray();
foreach (string file in files)
{
string fullName = name + "/" + Path.GetFileName(file);
if (regex.IsMatch(fullName))
{
if (unencodedRegex.IsMatch(fullName))
{
writer.Files.Add(
new PPeX.Subfile(
new FileSource(file),
Path.GetFileName(file),
name,
ArchiveFileType.Raw));
}
else
{
writer.Files.Add(
new PPeX.Subfile(
new FileSource(file),
Path.GetFileName(file),
name));
}
imported++;
}
}
Console.WriteLine("Imported " + imported + "/" + files.Length + " files");
}
}
int lastProgress = 0;
object progressLock = new object();
bool isUpdating = true;
Console.CursorVisible = false;
Progress <string> progressStatus = new Progress <string>(x =>
{
if (!isUpdating)
{
return;
}
lock (progressLock)
{
Console.SetCursorPosition(0, Console.CursorTop);
for (int i = 0; i < Console.WindowWidth - 1; i++)
{
Console.Write(" ");
}
Console.SetCursorPosition(0, Console.CursorTop);
Console.WriteLine(x.Trim());
Console.Write("[" + lastProgress + "% complete]");
}
});
Progress <int> progressPercentage = new Progress <int>(x =>
{
if (!isUpdating)
{
return;
}
lock (progressLock)
{
lastProgress = x;
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write("[" + lastProgress + "% complete]");
}
});
await using (FileStream fs = new FileStream(filename, FileMode.Create))
await writer.WriteAsync(fs, progressStatus, progressPercentage);
//wait for progress to update
while (lastProgress != 100)
{
await Task.Delay(50);
}
lock (progressPercentage)
{
isUpdating = false;
Console.CursorVisible = true;
}
}
private void btnTest_Click(object sender, EventArgs e)
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
//string dir = textBox1.Text;//@"C:\Users\Admin\Documents\GitHub\AA2Install\AA2Pack\bin\x86\Release\jg2p07_00_00";
foreach (ListViewItem item in lsvFiles.Items)
//ParallelOptions opt = new ParallelOptions();
//opt.MaxDegreeOfParallelism = 6;
//Parallel.ForEach(lsvFiles.Items.Cast<ListViewItem>(), opt, item =>
{
string entry = item.Text;
string name = Path.GetFileName(entry);
ExtendedArchiveWriter writer = null;
if (name.EndsWith(".ppx"))
{
using (FileStream fs = new FileStream(entry.Replace(".ppx", ".pp"), FileMode.Create))
{
byte[] buffer = PPeX.Utility.CreateHeader(new ExtendedArchive(entry));
fs.Write(buffer, 0, buffer.Length);
}
}
else if (name.EndsWith(".pp"))
{
writer = new ExtendedArchiveWriter(entry + "x", name);
ppParser pp = new ppParser(entry);
List <ppSubfile> read = pp.Subfiles.Cast <ppSubfile>().ToList();
/*Parallel.ForEach(read, subfile =>
* {
* var arc = new ArchiveFile(new PPSource(subfile), subfile.Name);
* lock (writer)
* {
* writer.Files.Add(arc);
* }
*
* });*/
foreach (IReadFile subfile in pp.Subfiles)
{
//writer.Files.Add(new ArchiveFile(new PPSource(subfile), subfile.Name));
Application.DoEvents();
}
}
else
{
writer = new ExtendedArchiveWriter(entry + ".ppx", name + ".pp");
foreach (string file in Directory.EnumerateFiles(entry))
{
if (file.EndsWith("ogg"))
{
continue;
}
// writer.Files.Add(new ArchiveFile(new FileSource(file), Path.GetFileName(file)));
Application.DoEvents();
}
}
sw.Stop();
//MessageBox.Show(sw.Elapsed.TotalMilliseconds.ToString());
sw.Restart();
if (writer != null)
{
writer.Write();
}
item.ForeColor = Color.ForestGreen;
Application.DoEvents();
//});
}
sw.Stop();
MessageBox.Show(sw.Elapsed.TotalMilliseconds.ToString());
}
public void Save()
{
if (txtArchiveName.Text == "" || txtSaveLocation.Text == "")
{
return;
}
ExtendedArchiveWriter writer = new ExtendedArchiveWriter(txtSaveLocation.Text, txtArchiveName.Text);
writer.DefaultCompression = (ArchiveFileCompression)cmbCompression.SelectedIndex;
writer.Type = (ArchiveType)cmbArchiveType.SelectedIndex;
IProgress <Tuple <string, int> > progress = new Progress <Tuple <string, int> >((x) =>
{
prgSaveProgress.Value = x.Item2;
txtSaveProg.AppendText(x.Item1);
});
Task.Run(() =>
{
try
{
progress.Report(new Tuple <string, int>(
"Performing first pass...\n",
0));
var allNodes = trvFiles.Nodes
.Cast <TreeNode>()
.SelectMany(GetNodeBranch);
int i = 1;
int total = allNodes.Count();
foreach (TreeNode node in allNodes)
{
if (node.Tag == null)
{
continue;
}
var holder = node.Tag as SubfileHolder;
writer.Files.Add(new ArchiveFile(holder.Source, node.FullPath, writer.DefaultCompression));
progress.Report(new Tuple <string, int>(
"",
100 * i++ / total));
}
writer.Write(progress);
btnSave.DynamicInvoke(() => btnSave.Enabled = true);
}
catch (Exception ex)
{
progress.Report(new Tuple <string, int>(
"ERROR: " + ex.Message + "\n",
100));
}
});
currentlyOpenedFile = Path.GetFileName(txtSaveLocation.Text);
IsModified = false;
}
