public static DLCSong ConvertDLCSong(DataArray songDta, GameArchives.IDirectory songRoot)
{
var path = songDta.Array("song").Array("name").String(1);
var hopoThreshold = songDta.Array("song").Array("hopo_threshold")?.Int(1) ?? 170;
var shortname = path.Split('/').Last();
var midPath = shortname + ".mid";
var artPath = $"gen/{shortname}_keep.png_xbox";
var miloPath = $"gen/{shortname}.milo_xbox";
var songId = songDta.Array("song_id").Node(1);
var name = songDta.Array("name").String(1);
var artist = songDta.Array("artist").String(1);
var mid = MidiCS.MidiFileReader.FromBytes(songRoot.GetFileAtPath(midPath).GetBytes());
// TODO: Catch possible conversion exceptions? i.e. Unsupported milo version
var milo = MiloFile.ReadFromStream(songRoot.GetFileAtPath(miloPath).GetStream());
var songData = SongDataConverter.ToSongData(songDta);
Texture.Texture artwork = null;
if (songData.AlbumArt)
{
artwork = Texture.TextureConverter.MiloPngToTexture(songRoot.GetFileAtPath(artPath).GetStream());
}
return(new DLCSong
{
SongData = songData,
Lipsync = LipsyncConverter.FromMilo(milo),
Mogg = songRoot.GetFile(shortname + ".mogg"),
MoggDta = MakeMoggDta(songDta),
MoggSong = DTX.FromDtaString($"(mogg_path \"{shortname}.mogg\")\r\n(midi_path \"{shortname}.rbmid\")\r\n"),
RBMidi = RBMidConverter.ToRBMid(mid, hopoThreshold),
Artwork = artwork,
RBSong = MakeRBSong(songDta)
});
}
public static Lipsync FromMilo(MiloFile milo)
{
var miloLips = milo.Entries
.Where(x => x is CharLipSync)
.Select(y => y as CharLipSync)
.ToList();
miloLips.Sort((x, y) =>
{
if (x.Name.Equals("song.lipsync", StringComparison.CurrentCultureIgnoreCase))
{
return(-1);
}
else if (y.Name.Equals("song.lipsync", StringComparison.CurrentCultureIgnoreCase))
{
return(1);
}
return(string.Compare(x.Name, y.Name));
});
int totalFrames = miloLips.Select(x => x.Frames.Length).DefaultIfEmpty(0).Max();
uint[] offsets = new uint[totalFrames + 1];
long currentOffset = 0;
var visemes = miloLips.SelectMany(x => x.Frames.SelectMany(y => y.Events))
.Select(z => z.VisemeName)
.Distinct()
.ToDictionary(a => a, b => (int)(currentOffset++));
using (MemoryStream ms = new MemoryStream())
{
for (int i = 0; i < totalFrames; i++)
{
currentOffset = ms.Position;
var data = GetBytes(miloLips, visemes, i);
ms.Write(data, 0, data.Length);
offsets[i + 1] = (uint)ms.Position;
}
return(new Lipsync
{
Version = 0,
Subtype = 0,
FrameRate = 30,
Visemes = visemes.Select((x, y) => x.Key).ToArray(),
Players = partMapping.Take(miloLips.Count).ToArray(),
FrameIndices = offsets,
FrameData = ms.ToArray()
});
}
}
public static MiloObjectDir OpenMiloFile(this AppState state, string miloPath)
{
MiloFile miloFile;
using (var fileStream = state.GetWorkingDirectory().GetStreamForFile(miloPath))
{
miloFile = MiloFile.ReadFromStream(fileStream);
}
var serializer = state.GetSerializer();
MiloObjectDir milo;
using (var miloStream = new MemoryStream(miloFile.Data))
{
milo = serializer.ReadFromStream <MiloObjectDir>(miloStream);
}
return(milo);
}
public void Parse(Ark2DirOptions op)
{
var scriptRegex = new Regex("(?i).((dtb)|(dta)|(([A-Z]+)(_dta_)([A-Z0-9]+)))$");
var scriptForgeRegex = new Regex("(?i)(_dta_)([A-Z0-9]+)$");
var csvRegex = new Regex("(?i).csv_([A-Z0-9]+)$");
var dtaRegex = new Regex("(?i).dta$");
var miloRegex = new Regex("(?i).milo(_[A-Z0-9]+)?$");
var genPathedFile = new Regex(@"(?i)(([^\/\\]+[\/\\])*)(gen[\/\\])([^\/\\]+)$");
var platformExtRegex = new Regex(@"(?i)_([A-Z0-9]+)$");
Archive ark;
int arkVersion;
bool arkEncrypted;
if (Directory.Exists(op.InputPath))
{
// Open as directory
ark = ArkFileSystem.FromDirectory(op.InputPath);
// TODO: Get from args probably
arkVersion = 10;
arkEncrypted = true;
}
else
{
// Open as ark
var arkFile = ArkFile.FromFile(op.InputPath);
arkVersion = (int)arkFile.Version;
arkEncrypted = arkFile.Encrypted;
ark = arkFile;
}
var scriptsToConvert = ark.Entries
.Where(x => op.ConvertScripts &&
scriptRegex.IsMatch(x.FullPath))
.ToList();
var csvsToConvert = ark.Entries
.Where(x => op.ConvertScripts &&
csvRegex.IsMatch(x.FullPath))
.ToList();
var milosToInflate = ark.Entries
.Where(x => op.InflateMilos &&
miloRegex.IsMatch(x.FullPath))
.ToList();
var entriesToExtract = ark.Entries
.Where(x => op.ExtractAll)
.Except(scriptsToConvert)
.Except(milosToInflate)
.ToList();
foreach (var arkEntry in entriesToExtract)
{
var filePath = ExtractEntry(ark, arkEntry, CombinePath(op.OutputPath, arkEntry.FullPath));
Console.WriteLine($"Wrote \"{filePath}\"");
}
// Create temp path
var tempDir = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, true);
}
foreach (var miloEntry in milosToInflate)
{
var filePath = ExtractEntry(ark, miloEntry, CombinePath(op.OutputPath, miloEntry.FullPath));
// Inflate milo
var milo = MiloFile.ReadFromFile(filePath);
milo.Structure = BlockStructure.MILO_A;
milo.WriteToFile(filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
foreach (var csvEntry in csvsToConvert)
{
var csvStream = ark.GetArkEntryFileStream(csvEntry);
var csv = CSVFile.FromForgeCSVStream(csvStream);
// Write to file
var csvPath = CombinePath(op.OutputPath, csvEntry.FullPath);
csvPath = platformExtRegex.Replace(csvPath, "");
csv.SaveToFileAsCSV(csvPath);
Console.WriteLine($"Wrote \"{csvPath}\"");
}
var successDtas = 0;
foreach (var scriptEntry in scriptsToConvert)
{
// Just extract file if dta script
if (dtaRegex.IsMatch(scriptEntry.FullPath))
{
var filePath = ExtractEntry(ark, scriptEntry, CombinePath(op.OutputPath, scriptEntry.FullPath));
Console.WriteLine($"Wrote \"{filePath}\"");
continue;
}
// Creates output path
var dtaPath = CombinePath(op.OutputPath, scriptEntry.FullPath);
dtaPath = !scriptForgeRegex.IsMatch(dtaPath)
? $"{dtaPath.Substring(0, dtaPath.Length - 1)}a" // Simply change b -> a
: scriptForgeRegex.Replace(dtaPath, "");
// Removes gen sub directory
if (genPathedFile.IsMatch(dtaPath))
{
var match = genPathedFile.Match(dtaPath);
dtaPath = $"{match.Groups[1]}{match.Groups[4]}";
}
var tempDtbPath = ExtractEntry(ark, scriptEntry, Path.Combine(tempDir, Path.GetRandomFileName()));
try
{
ScriptHelper.ConvertDtbToDta(tempDtbPath, tempDir, arkEncrypted, arkVersion, dtaPath);
Console.WriteLine($"Wrote \"{dtaPath}\"");
successDtas++;
}
catch (DTBParseException ex)
{
Console.WriteLine($"Unable to convert to script, skipping \'{scriptEntry.FullPath}\'");
if (File.Exists(dtaPath))
{
File.Delete(dtaPath);
}
}
catch (Exception ex)
{
}
}
if (scriptsToConvert.Count > 0)
{
Console.WriteLine($"Converted {successDtas} of {scriptsToConvert.Count} scripts");
}
// Clean up temp files
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, true);
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Usage();
return;
}
void WithIO(Action <Stream, Stream> action)
{
using (var fi = File.OpenRead(args[1]))
using (var fo = File.OpenWrite(args[2]))
action(fi, fo);
}
var makepkg = true;
switch (args[0])
{
case "rbmid2mid":
WithIO((fi, fo) =>
{
var rbmid = RBMidReader.ReadStream(fi);
var midi = RBMidConverter.ToMid(rbmid);
MidiCS.MidiFileWriter.WriteSMF(midi, fo);
});
break;
case "mid2rbmid":
WithIO((fi, fo) =>
{
var mid = MidiCS.MidiFileReader.FromStream(fi);
var rbmid = RBMidConverter.ToRBMid(mid);
RBMidWriter.WriteStream(rbmid, fo);
});
break;
case "reprocess":
WithIO((fi, fo) =>
{
var rbmid = RBMidReader.ReadStream(fi);
var processed = RBMidConverter.ToRBMid(RBMidConverter.ToMid(rbmid), rbmid.HopoThreshold);
RBMidWriter.WriteStream(processed, fo);
});
break;
case "tex2png":
WithIO((fi, fo) =>
{
var tex = TextureReader.ReadStream(fi);
var bitmap = TextureConverter.ToBitmap(tex, 0);
bitmap.Save(fo, System.Drawing.Imaging.ImageFormat.Png);
});
break;
case "png2tex":
WithIO((fi, fo) =>
{
var img = System.Drawing.Image.FromStream(fi);
var tex = TextureConverter.ToTexture(img);
TextureWriter.WriteStream(tex, fo);
});
break;
case "milopng2tex":
WithIO((fi, fo) =>
{
var tex = TextureConverter.MiloPngToTexture(fi);
TextureWriter.WriteStream(tex, fo);
});
break;
case "mesh2obj":
{
var input = args[1];
var output = args[2];
using (var fi = File.OpenRead(input))
{
var mesh = HxMeshReader.ReadStream(fi);
var obj = HxMeshConverter.ToObj(mesh);
File.WriteAllText(output, obj);
}
break;
}
case "milo2lip":
case "milo2lips":
case "milo2lipsync":
WithIO((fi, fo) =>
{
var milo = MiloFile.ReadFromStream(fi);
var lipsync = LipsyncConverter.FromMilo(milo);
new LipsyncWriter(fo).WriteStream(lipsync);
});
break;
case "version":
var assembly = System.Reflection.Assembly.GetExecutingAssembly();
var version = FileVersionInfo.GetVersionInfo(assembly.Location).FileVersion;
var libAssembly = System.Reflection.Assembly.GetAssembly(typeof(RBMid));
var libVersion = FileVersionInfo.GetVersionInfo(libAssembly.Location).FileVersion;
Console.WriteLine($"ForgeTool v{version}");
Console.WriteLine($"LibForge v{libVersion}");
break;
case "test":
{
var dir = args[1];
int succ = 0, warn = 0, fail = 0;
var files = dir.EndsWith(".rbmid_ps4") || dir.EndsWith(".rbmid_pc") ?
new[] { dir } : Directory.EnumerateFiles(dir, "*.rbmid_*");
foreach (var f in files)
{
var info = new FileInfo(f);
var name = info.Name;
using (var fi = File.OpenRead(f))
{
try
{
var rbmid = RBMidReader.ReadStream(fi);
var midi = RBMidConverter.ToMid(rbmid);
var rbmid2 = RBMidConverter.ToRBMid(midi, rbmid.HopoThreshold);
using (var ms = new MemoryStream((int)fi.Length))
{
// TODO: Switch this to rbmid2 once we are generating all events
// (regardless of whether the event data are all correct)
RBMidWriter.WriteStream(rbmid, ms);
ms.Position = 0;
if (ms.Length == fi.Length)
{
var comparison = rbmid.Compare(rbmid2);
if (comparison != null)
{
throw new CompareException("File comparison failed at field: " + comparison);
}
Console.WriteLine($"[OK] {name}");
succ++;
}
else
{
Console.Write($"[WARN] {name}:");
Console.WriteLine($" Processed file had different length ({fi.Length} orig, {ms.Length} processed)");
warn++;
}
}
} catch (CompareException e)
{
Console.WriteLine($"[WARN] {name}: {e.Message}");
warn++;
}
catch (Exception e)
{
Console.WriteLine($"[ERROR] {name}: {e.Message}");
Console.WriteLine(e.StackTrace);
fail++;
}
}
}
Console.WriteLine($"Summary: {succ} OK, {warn} WARN, {fail} ERROR");
if (warn > 0)
{
Console.WriteLine("(a = converted file, b = original)");
}
}
break;
case "simpletest":
{
var dir = args[1];
int succ = 0, warn = 0, fail = 0;
foreach (var f in Directory.EnumerateFiles(dir, "*.rbmid_*"))
{
var info = new FileInfo(f);
var name = info.Name;
using (var fi = File.OpenRead(f))
{
try
{
var rbmid = RBMidReader.ReadStream(fi);
using (var ms = new MemoryStream((int)fi.Length))
{
RBMidWriter.WriteStream(rbmid, ms);
ms.Position = 0;
if (ms.Length == fi.Length)
{
succ++;
}
else
{
Console.WriteLine($"[WARN] {name}:");
Console.WriteLine($" Processed file had different length ({fi.Length} orig, {ms.Length} processed)");
warn++;
}
}
}
catch (Exception e)
{
Console.WriteLine($"[ERROR] {name}:");
Console.WriteLine(" " + e.Message);
fail++;
}
}
}
Console.WriteLine($"Summary: {succ} OK, {warn} WARN, {fail} ERROR");
}
break;
case "testrbsong":
{
var dir = args[1];
int succ = 0, warn = 0, fail = 0;
var files = dir.EndsWith(".rbsong") ?
new[] { dir } : Directory.EnumerateFiles(dir, "*.rbsong");
foreach (var f in files)
{
var info = new FileInfo(f);
var name = info.Name;
using (var fi = File.OpenRead(f))
{
try
{
var rbsong = new RBSongReader(fi).Read();
using (var ms = new MemoryStream((int)fi.Length))
{
new RBSongWriter(ms).WriteStream(rbsong);
ms.Position = 0;
if (ms.Length == fi.Length)
{
succ++;
}
else
{
Console.WriteLine($"[WARN] {name}:");
Console.WriteLine($" Processed file had different length ({fi.Length} orig, {ms.Length} processed)");
warn++;
}
}
}
catch (Exception e)
{
Console.WriteLine($"[ERROR] {name}:");
Console.WriteLine(" " + e.Message);
fail++;
}
}
}
Console.WriteLine($"Summary: {succ} OK, {warn} WARN, {fail} ERROR");
if (fail > 0)
{
Console.WriteLine("(a = converted file, b = original)");
}
}
break;
case "rbsong2rbsong":
WithIO((i, o) => new RBSongWriter(o).WriteStream(new RBSongReader(i).Read()));
break;
case "con2gp4":
makepkg = false;
goto case "con2pkg";
case "con2pkg":
{
var i = 0;
bool eu = false;
string pkgId = null, pkgDesc = null;
while (++i < args.Length - 4)
{
switch (args[i])
{
case "--scee":
eu = true;
continue;
case "--id":
pkgId = args[++i];
continue;
case "--desc":
pkgDesc = args[++i];
continue;
}
break;
}
if (makepkg)
{
var cmd = args[i++];
var con = args[i++];
var dest = args[i++];
var tmpDir = Path.Combine(Path.GetTempPath(), "forgetool_tmp_build");
if (!Directory.Exists(con))
{
var conFilename = Path.GetFileName(con);
PkgCreator.ConToGp4(con, tmpDir, eu, pkgId, pkgDesc);
}
else
{
PkgCreator.ConsToGp4(con, tmpDir, eu, pkgId, pkgDesc);
}
PkgCreator.BuildPkg(cmd, Path.Combine(tmpDir, "project.gp4"), dest);
Directory.Delete(tmpDir, true);
}
else
{
var con = args[i++];
var dest = args[i++];
PkgCreator.ConToGp4(con, dest, eu, pkgId, pkgDesc);
}
}
break;
default:
Usage();
break;
}
}
public static object LoadObject(GameArchives.IFile i)
{
if (i.Name.Contains(".bmp_") || i.Name.Contains(".png_"))
{
using (var s = i.GetStream())
{
try
{
return(TextureReader.ReadStream(s));
}
catch (Exception ex)
{
System.Windows.Forms.MessageBox.Show("Couldn't load texture: " + ex.Message);
return(null);
}
}
}
else if (i.Name.Contains("_dta_") || i.Name.EndsWith(".dtb"))
{
using (var s = i.GetStream())
{
var data = DtxCS.DTX.FromDtb(s);
var sb = new StringBuilder();
foreach (var x in data.Children)
{
sb.AppendLine(x.ToString(0));
}
return(sb.ToString());
}
}
else if (i.Name.EndsWith(".dta") || i.Name.EndsWith(".moggsong"))
{
using (var s = i.GetStream())
using (var r = new System.IO.StreamReader(s))
{
return(r.ReadToEnd());
}
}
else if (i.Name.Contains(".songdta"))
{
using (var s = i.GetStream())
{
var songData = SongDataReader.ReadStream(s);
return(songData);
}
}
else if (i.Name.Contains(".fbx"))
{
using (var s = i.GetStream())
{
return(HxMeshReader.ReadStream(s));
}
}
else if (i.Name.Contains(".rbmid_"))
{
using (var s = i.GetStream())
{
return(RBMidReader.ReadStream(s));
}
}
else if (i.Name.Contains(".lipsync"))
{
using (var s = i.GetStream())
{
return(new LipsyncReader(s).Read());
}
}
else if (i.Name.Contains(".rbsong"))
{
using (var s = i.GetStream())
{
var rbsong = new RBSongResource();
rbsong.Load(s);
return(rbsong);
}
}
else if (i.Name.Contains(".gp4"))
{
using (var s = i.GetStream())
{
return(LibOrbisPkg.GP4.Gp4Project.ReadFrom(s));
}
}
else if (i.Name.Contains(".pkg"))
{
using (var s = i.GetStream())
{
return(new LibOrbisPkg.PKG.PkgReader(s).ReadHeader());
}
}
else if (i.Name.EndsWith(".milo_xbox"))
{
using (var s = i.GetStream())
{
return(MiloFile.ReadFromStream(s));
}
}
else if (i.Name.Contains(".csv"))
{
using (var s = i.GetStream())
{
return(CsvData.LoadFile(s).ToString());
}
}
else if (i.Name.EndsWith(".uexp"))
{
using (var s = i.GetStream())
{
return(LibForge.Fuser.FuserAsset.Read(s));
}
}
else
{
return(null);
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Usage();
return;
}
void WithIO(Action <Stream, Stream> action)
{
using (var fi = File.OpenRead(args[1]))
using (var fo = File.OpenWrite(args[2]))
action(fi, fo);
}
var makepkg = true;
switch (args[0])
{
case "rbmid2mid":
WithIO((fi, fo) =>
{
var rbmid = RBMidReader.ReadStream(fi);
var midi = RBMidConverter.ToMid(rbmid);
MidiCS.MidiFileWriter.WriteSMF(midi, fo);
});
break;
case "mid2rbmid":
WithIO((fi, fo) =>
{
var mid = MidiCS.MidiFileReader.FromStream(fi);
var rbmid = RBMidConverter.ToRBMid(mid);
RBMidWriter.WriteStream(rbmid, fo);
});
break;
case "reprocess":
WithIO((fi, fo) =>
{
var rbmid = RBMidReader.ReadStream(fi);
var processed = RBMidConverter.ToRBMid(RBMidConverter.ToMid(rbmid), rbmid.HopoThreshold);
RBMidWriter.WriteStream(processed, fo);
});
break;
case "tex2png":
WithIO((fi, fo) =>
{
var tex = TextureReader.ReadStream(fi);
var bitmap = TextureConverter.ToBitmap(tex, 0);
bitmap.Save(fo, System.Drawing.Imaging.ImageFormat.Png);
});
break;
case "png2tex":
WithIO((fi, fo) =>
{
var img = System.Drawing.Image.FromStream(fi);
var tex = TextureConverter.ToTexture(img);
TextureWriter.WriteStream(tex, fo);
});
break;
case "milopng2tex":
WithIO((fi, fo) =>
{
var tex = TextureConverter.MiloPngToTexture(fi);
TextureWriter.WriteStream(tex, fo);
});
break;
case "mesh2obj":
{
var input = args[1];
var output = args[2];
using (var fi = File.OpenRead(input))
{
var mesh = HxMeshReader.ReadStream(fi);
var obj = HxMeshConverter.ToObj(mesh);
File.WriteAllText(output, obj);
}
break;
}
case "milo2lip":
case "milo2lips":
case "milo2lipsync":
WithIO((fi, fo) =>
{
var milo = MiloFile.ReadFromStream(fi);
var lipsync = LipsyncConverter.FromMilo(milo);
new LipsyncWriter(fo).WriteStream(lipsync);
});
break;
case "version":
var assembly = System.Reflection.Assembly.GetExecutingAssembly();
var version = FileVersionInfo.GetVersionInfo(assembly.Location).FileVersion;
var libAssembly = System.Reflection.Assembly.GetAssembly(typeof(RBMid));
var libVersion = FileVersionInfo.GetVersionInfo(libAssembly.Location).FileVersion;
Console.WriteLine($"ForgeTool v{version}");
Console.WriteLine($"LibForge v{libVersion}");
break;
case "test":
{
var dir = args[1];
int succ = 0, warn = 0, fail = 0;
var files = dir.EndsWith(".rbmid_ps4") || dir.EndsWith(".rbmid_pc") ?
new[] { dir } : Directory.EnumerateFiles(dir, "*.rbmid_*");
foreach (var f in files)
{
var info = new FileInfo(f);
var name = info.Name;
using (var fi = File.OpenRead(f))
{
try
{
var rbmid = RBMidReader.ReadStream(fi);
var midi = RBMidConverter.ToMid(rbmid);
var rbmid2 = RBMidConverter.ToRBMid(midi, rbmid.HopoThreshold);
using (var ms = new MemoryStream((int)fi.Length))
{
// TODO: Switch this to rbmid2 once we are generating all events
// (regardless of whether the event data are all correct)
RBMidWriter.WriteStream(rbmid, ms);
ms.Position = 0;
if (ms.Length == fi.Length)
{
var comparison = rbmid.Compare(rbmid2);
if (comparison != null)
{
throw new CompareException("File comparison failed at field: " + comparison);
}
Console.WriteLine($"[OK] {name}");
succ++;
}
else
{
Console.Write($"[WARN] {name}:");
Console.WriteLine($" Processed file had different length ({fi.Length} orig, {ms.Length} processed)");
warn++;
}
}
} catch (CompareException e)
{
Console.WriteLine($"[WARN] {name}: {e.Message}");
warn++;
}
catch (Exception e)
{
Console.WriteLine($"[ERROR] {name}: {e.Message}");
Console.WriteLine(e.StackTrace);
fail++;
}
}
}
Console.WriteLine($"Summary: {succ} OK, {warn} WARN, {fail} ERROR");
if (warn > 0)
{
Console.WriteLine("(a = converted file, b = original)");
}
if (warn > 0 || fail > 0)
{
Environment.Exit(1);
}
}
break;
case "simpletest":
{
var dir = args[1];
int succ = 0, warn = 0, fail = 0;
foreach (var f in Directory.EnumerateFiles(dir, "*.rbmid_*"))
{
var info = new FileInfo(f);
var name = info.Name;
using (var fi = File.OpenRead(f))
{
try
{
var rbmid = RBMidReader.ReadStream(fi);
using (var ms = new MemoryStream((int)fi.Length))
{
RBMidWriter.WriteStream(rbmid, ms);
ms.Position = 0;
if (ms.Length == fi.Length)
{
succ++;
}
else
{
Console.WriteLine($"[WARN] {name}:");
Console.WriteLine($" Processed file had different length ({fi.Length} orig, {ms.Length} processed)");
warn++;
}
}
}
catch (Exception e)
{
Console.WriteLine($"[ERROR] {name}:");
Console.WriteLine(" " + e.Message);
fail++;
}
}
}
Console.WriteLine($"Summary: {succ} OK, {warn} WARN, {fail} ERROR");
}
break;
case "testrbsong":
{
var dir = args[1];
int succ = 0, warn = 0, fail = 0;
var files = dir.EndsWith(".rbsong") ?
new[] { dir } : Directory.EnumerateFiles(dir, "*.rbsong");
foreach (var f in files)
{
var info = new FileInfo(f);
var name = info.Name;
using (var fi = File.OpenRead(f))
{
try
{
var rbsong = new RBSongResource();
rbsong.Load(fi);
using (var ms = new MemoryStream((int)fi.Length))
{
new RBSongResourceWriter(ms).WriteStream(rbsong);
ms.Position = 0;
if (ms.Length == fi.Length)
{
succ++;
}
else
{
Console.WriteLine($"[WARN] {name}:");
Console.WriteLine($" Processed file had different length ({fi.Length} orig, {ms.Length} processed)");
warn++;
}
}
}
catch (Exception e)
{
Console.WriteLine($"[ERROR] {name}:");
Console.WriteLine(" " + e.Message);
fail++;
}
}
}
Console.WriteLine($"Summary: {succ} OK, {warn} WARN, {fail} ERROR");
if (fail > 0)
{
Console.WriteLine("(a = converted file, b = original)");
}
}
break;
case "rbsong2rbsong":
WithIO((i, o) => {
var rbsong = new RBSongResource();
rbsong.Load(i);
new RBSongResourceWriter(o).WriteStream(rbsong);
});
break;
case "con2gp4":
makepkg = false;
goto case "con2pkg";
case "con2pkg":
{
var i = 0;
bool eu = false;
string pkgId = null, pkgDesc = null;
while (++i < args.Length - 3)
{
switch (args[i])
{
case "--scee":
eu = true;
continue;
case "--id":
pkgId = args[++i];
continue;
case "--desc":
pkgDesc = args[++i];
continue;
}
break;
}
if (makepkg)
{
var con = args[i++];
var dest = args[i++];
var tmpDir = Path.Combine(Path.GetTempPath(), "forgetool_tmp_build");
if (!Directory.Exists(con))
{
var conFilename = Path.GetFileName(con);
PkgCreator.ConToGp4(con, tmpDir, eu, pkgId, pkgDesc);
}
else
{
PkgCreator.ConsToGp4(con, tmpDir, eu, pkgId, pkgDesc);
}
PkgCreator.BuildPkg(Path.Combine(tmpDir, "project.gp4"), dest);
Directory.Delete(tmpDir, true);
}
else
{
var con = args[i++];
var dest = args[i++];
PkgCreator.ConToGp4(con, dest, eu, pkgId, pkgDesc);
}
}
break;
case "csv2txt":
{
var input = args[1];
var output = args[2];
using (var fi = File.OpenRead(input))
{
var csv = CsvData.LoadFile(fi);
File.WriteAllText(output, csv.ToString());
}
}
break;
case "arkorder":
{
var input = args[1];
var output = args[2];
var sb = new StringBuilder();
var archive = new Archive(input);
using (var o = File.OpenWrite(output))
using (var sw = new StreamWriter(o))
{
archive.WriteArkorder(sw);
}
}
break;
case "arkbuild":
{
var filedir = args[1];
var arkorder = args[2];
var outdir = args[3];
var name = args[4];
if (!Directory.Exists(filedir))
{
Console.WriteLine($"Error: {filedir} does not exist.");
return;
}
if (!Directory.Exists(outdir))
{
Console.WriteLine($"Error: {outdir} does not exist.");
return;
}
if (!File.Exists(arkorder))
{
Console.WriteLine($"Error: {arkorder} not found.");
return;
}
var arks = new List <List <Tuple <string, IFile> > >();
DataArray arkOrder;
using (var order = File.OpenRead(arkorder))
using (var reader = new StreamReader(order))
{
arkOrder = DtxCS.DTX.FromDtaStream(order);
}
int arkIndex = 0;
arks.Add(new List <Tuple <string, IFile> >());
foreach (var x in arkOrder.Children)
{
if (x is DataSymbol s)
{
var fullPath = Path.Combine(filedir, s.Name.Replace('/', Path.DirectorySeparatorChar));
if (!File.Exists(fullPath))
{
Console.WriteLine($"Error: {fullPath} could not be found.");
return;
}
IFile f = Util.LocalFile(fullPath);
arks[arkIndex].Add(Tuple.Create(s.Name, f));
}
else if (x is DataCommand c && c.Symbol(0).Name == "split_ark")
{
arkIndex++;
arks[arkIndex] = new List <Tuple <string, IFile> >();
}
}
var builder = new ArkBuilder(name, arks);
Console.Write($"Writing {name}.hdr and ark files to {outdir}...");
builder.Save(outdir);
}
break;
default:
Usage();
break;
}
}
public IActionResult ExtractFilesFromArk([FromBody] ScanRequest request, bool extractMilos, bool extractDTAs, bool convertTextures)
{
Archive ark;
if (!System.IO.File.Exists(request.InputPath))
{
// Open as directory if available
if (Directory.Exists(request.InputPath))
{
ark = ArkFileSystem.FromDirectory(request.InputPath);
}
else
{
return(BadRequest($"File \"{request.InputPath}\" does not exist!"));
}
}
else
{
// Open as archive
ark = ArkFile.FromFile(request.InputPath);
}
if (request.OutputPath == null)
{
return(BadRequest($"Output directory cannot be null!"));
}
string CombinePath(string basePath, string path)
{
// Consistent slash
basePath = (request.OutputPath ?? "").Replace("/", "\\");
path = (path ?? "").Replace("/", "\\");
Regex dotRegex = new Regex(@"[.]+[\\]");
if (dotRegex.IsMatch(path))
{
// Replaces dotdot path
path = dotRegex.Replace(path, x => $"({x.Value.Substring(0, x.Value.Length - 1)})\\");
}
return(Path.Combine(basePath, path));
}
string GetNonGenPath(string path)
{
// Consistent slash
path = (path ?? "").Replace("/", "\\");
Regex genRegex = new Regex(@"gen\\[^\\]+$", RegexOptions.IgnoreCase);
Regex platformRegex = new Regex(@"_[^_]+$", RegexOptions.IgnoreCase); // TODO: Revisit for Forge extensions
if (genRegex.IsMatch(path))
{
var splitPath = path.Split('\\');
var dir = string.Join("\\", splitPath.SkipLast(2));
var file = platformRegex.Replace(splitPath.Last(), "");
path = $"{dir}\\{file}";
}
return(path);
}
void SaveAsFile(MiloObjectBytes miloEntry, string basePath)
{
var fileName = SanitizeFileName(miloEntry.Name);
var filePath = basePath = Path.Combine(basePath, miloEntry.Type, fileName);
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
System.IO.File.WriteAllBytes(filePath, miloEntry.Data);
Console.WriteLine($"Wrote \"{filePath}\"");
}
// Extract everything
if (!extractDTAs && !extractMilos && !convertTextures)
{
foreach (var arkEntry in ark.Entries)
{
var filePath = CombinePath(request.OutputPath, arkEntry.FullPath);
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
using (var fs = System.IO.File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Write))
{
using (var stream = ark.GetArkEntryFileStream(arkEntry))
{
stream.CopyTo(fs);
}
}
Console.WriteLine($"Wrote \"{filePath}\"");
}
return(Ok());
}
string SanitizeFileName(string fileName)
{
// Sanitize file name
var invalidChars = new Regex($"[{new string(Path.GetInvalidFileNameChars())}]", RegexOptions.IgnoreCase);
return(invalidChars.Replace(fileName, ""));
}
void ProcessMiloArkEntry(ArkEntry miloArkEntry)
{
var filePath = CombinePath(request.OutputPath, GetNonGenPath(miloArkEntry.FullPath));
using (var stream = ark.GetArkEntryFileStream(miloArkEntry))
{
var milo = MiloFile.ReadFromStream(stream);
var miloSerializer = new MiloSerializer(new SystemInfo()
{
Version = milo.Version,
BigEndian = milo.BigEndian,
Platform = Enum.Parse <Platform>(request.Platform)
});
var miloDir = new MiloObjectDir();
using (var ms = new MemoryStream(milo.Data))
{
miloSerializer.ReadFromStream(ms, miloDir);
}
if (convertTextures)
{
var textureEntries = miloDir.Entries
.Where(x => x.Type == "Tex")
.Select(x => x is Tex ? x as Tex : miloSerializer.ReadFromMiloObjectBytes <Tex>(x as MiloObjectBytes))
.Where(x => x.Bitmap != null && x.Bitmap.RawData?.Length > 0)
.ToList();
if (textureEntries.Count <= 0)
{
return;
}
foreach (var texEntry in textureEntries)
{
var entryName = Path.GetFileNameWithoutExtension(SanitizeFileName(texEntry.Name));
var pngName = $"{entryName}.png";
var pngPath = Path.Combine(filePath, texEntry.Type, pngName);
texEntry.Bitmap.SaveAs(miloSerializer.Info, pngPath);
Console.WriteLine($"Wrote \"{pngPath}\"");
// Write DTA script to file
var scriptName = texEntry?.ScriptName ?? "";
if (texEntry.Script != null)
{
var dtaName = (string.IsNullOrEmpty(scriptName))
? $"{entryName}.dta"
: $"{entryName}_{scriptName}.dta";
var dtaPath = Path.Combine(filePath, texEntry.Type, dtaName);
var parent = new ParentItem(ParentType.Default);
foreach (var item in texEntry.Script.Items)
{
parent.Add(item);
}
texEntry.Script.Items.Clear();
texEntry.Script.Items.Add(parent);
System.IO.File.WriteAllText(dtaPath, texEntry.Script.ToString());
Console.WriteLine($"Wrote \"{dtaPath}\"");
}
}
}
if (extractMilos)
{
if (miloDir.Entries.Count <= 0)
{
return;
}
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
// Saves milo entries
miloDir.Entries.ForEach(x =>
{
SaveAsFile(x as MiloObjectBytes, filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
});
if (miloDir.Extras.ContainsKey("DirectoryEntry"))
{
SaveAsFile(miloDir.Extras["DirectoryEntry"] as MiloObjectBytes, filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
}
}
}
void ProcessBitmapArkEntry(ArkEntry bitmapArkEntry)
{
var arkPath = GetNonGenPath(bitmapArkEntry.FullPath);
var filePath = CombinePath(request.OutputPath, $"{Path.GetDirectoryName(arkPath)}\\{Path.GetFileNameWithoutExtension(bitmapArkEntry.FileName)}.png");
using (var stream = ark.GetArkEntryFileStream(bitmapArkEntry))
{
var miloSerializer = new MiloSerializer(new SystemInfo()
{
Version = 25, // 10 = gh1, 24 = gh2
BigEndian = false, // Even on PPC it's LE
Platform = Enum.Parse <Platform>(request.Platform)
});
var bitmap = miloSerializer.ReadFromStream <HMXBitmap>(stream);
bitmap.SaveAs(miloSerializer.Info, filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
}
// Extract milos
if (extractMilos || convertTextures)
{
var miloEntries = ark.Entries.Where(x => _miloRegex.IsMatch(x.FullPath));
Parallel.ForEach(miloEntries, (entry) =>
{
ProcessMiloArkEntry(entry);
});
//foreach (var entry in miloEntries) ProcessMiloArkEntry(entry);
}
// Convert textures
if (convertTextures)
{
var bitmapArkEntries = ark.Entries.Where(x => _bitmapRegex.IsMatch(x.FullPath));
Parallel.ForEach(bitmapArkEntries, (entry) =>
{
ProcessBitmapArkEntry(entry);
});
//foreach (var entry in bitmapArkEntries) ProcessBitmapArkEntry(entry);
}
return(Ok());
}
public IActionResult ScanArkPost([FromBody] ScanRequest request)
{
_miloContext.Database.EnsureCreated();
var sw = Stopwatch.StartNew();
// Updates games (arks)
var game = _miloContext.Arks.FirstOrDefault(x => x.Title == request.GameTitle &&
x.Platform == request.Platform &&
x.Region == request.Region);
if (game == null)
{
// Create game
game = new Data.MiloEntities.Ark()
{
Title = request.GameTitle,
Platform = request.Platform,
Region = request.Region
};
_miloContext.Arks.Add(game);
_miloContext.SaveChanges();
}
var ark = ArkFile.FromFile(request.InputPath);
game.ArkVersion = (int)ark.Version;
var miloEntries = new List <Data.MiloEntities.ArkEntry>();
var totalMiloEntries = 0;
// Updates ark entries
foreach (var arkEntry in ark.Entries)
{
var entry = arkEntry as OffsetArkEntry;
var contextEntry = _miloContext.ArkEntries.FirstOrDefault(x => x.Ark == game && x.Path == entry.FullPath);
if (contextEntry == null)
{
contextEntry = new Data.MiloEntities.ArkEntry()
{
Ark = game,
Path = entry.FullPath
};
_miloContext.ArkEntries.Add(contextEntry);
_miloContext.SaveChanges();
}
contextEntry.Part = entry.Part;
contextEntry.Offset = entry.Offset;
contextEntry.Size = (int)entry.Size;
contextEntry.InflatedSize = (int)entry.InflatedSize;
if (_miloRegex.IsMatch(contextEntry.Path))
{
miloEntries.Add(contextEntry);
}
_miloContext.Update(contextEntry);
}
// Updates milos
foreach (var miloEntry in miloEntries)
{
var arkEntry = ark.Entries.First(x => x.FullPath == miloEntry.Path);
var mf = MiloFile.ReadFromStream(ark.GetArkEntryFileStream(arkEntry));
var serializer = new MiloSerializer(new SystemInfo()
{
BigEndian = mf.BigEndian, Version = mf.Version
});
MiloObjectDir milo;
using (var ms = new MemoryStream(mf.Data))
{
milo = serializer.ReadFromStream <MiloObjectDir>(ms);
}
totalMiloEntries += milo.Entries.Count;
var contextEntry = _miloContext.Milos.FirstOrDefault(x => x.ArkEntry == miloEntry);
if (contextEntry == null)
{
contextEntry = new Data.MiloEntities.Milo()
{
ArkEntry = miloEntry
};
_miloContext.Milos.Add(contextEntry);
_miloContext.SaveChanges();
}
contextEntry.Version = mf.Version;
contextEntry.TotalSize = mf.Data.Length;
contextEntry.Name = milo.Name ?? "";
contextEntry.Type = milo.Type ?? "";
var dirEntry = milo.Entries
.Where(x => ((string)x.Type).EndsWith("Dir") && x is MiloObjectBytes)
.Select(x => x as MiloObjectBytes)
.FirstOrDefault();
if (dirEntry != null)
{
contextEntry.Size = dirEntry.Data.Length;
contextEntry.Magic = dirEntry.GetMagic();
}
else
{
contextEntry.Size = -1;
contextEntry.Magic = -1;
}
// Updates milo entries
foreach (var mEntry in milo.Entries.Where(x => x is MiloObjectBytes && x != dirEntry).Select(y => y as MiloObjectBytes))
{
var contextMEntry = _miloContext.MiloEntries.FirstOrDefault(x => x.Milo == contextEntry && x.Name == mEntry.Name && x.Type == mEntry.Type);
if (contextMEntry == null)
{
contextMEntry = new Data.MiloEntities.MiloEntry()
{
Milo = contextEntry
};
_miloContext.MiloEntries.Add(contextMEntry);
_miloContext.SaveChanges();
}
contextMEntry.Name = mEntry.Name ?? "";
contextMEntry.Type = mEntry.Type ?? "";
contextMEntry.Size = mEntry.Data.Length;
contextMEntry.Magic = mEntry.GetMagic();
_miloContext.Update(contextMEntry);
}
_miloContext.Update(contextEntry);
}
_miloContext.SaveChanges();
sw.Stop();
return(Ok(new ScanResult()
{
TotalArkEntries = ark.Entries.Count,
TotalMilos = miloEntries.Count,
TotalMiloEntries = totalMiloEntries,
TimeElapsed = sw.ElapsedMilliseconds
}));
}
public void Parse(Ark2DirOptions op)
{
var scriptRegex = new Regex("(?i).((dtb)|(dta)|(([A-Z]+)(_dta_)([A-Z0-9]+)))$");
var scriptForgeRegex = new Regex("(?i)(_dta_)([A-Z0-9]+)$");
var csvRegex = new Regex("(?i).csv_([A-Z0-9]+)$");
var dtaRegex = new Regex("(?i).dta$");
var textureRegex = new Regex("(?i).((bmp)|(png))(_[A-Z0-9]+)$");
var miloRegex = new Regex("(?i).((gh)|(milo)|(rnd))(_[A-Z0-9]+)?$");
var genPathedFile = new Regex(@"(?i)(([^\/\\]+[\/\\])*)(gen[\/\\])([^\/\\]+)$");
var platformExtRegex = new Regex(@"(?i)_([A-Z0-9]+)$");
Archive ark;
int arkVersion;
bool arkEncrypted;
if (Directory.Exists(op.InputPath))
{
// Open as directory
ark = ArkFileSystem.FromDirectory(op.InputPath);
// TODO: Get from args probably
arkVersion = 10;
arkEncrypted = true;
}
else
{
// Open as ark
var arkFile = ArkFile.FromFile(op.InputPath);
arkVersion = (int)arkFile.Version;
arkEncrypted = arkFile.Encrypted;
ark = arkFile;
}
var scriptsToConvert = ark.Entries
.Where(x => op.ConvertScripts &&
arkVersion >= 3 && // Amp dtbs not supported right now
scriptRegex.IsMatch(x.FullPath))
.ToList();
var csvsToConvert = ark.Entries
.Where(x => op.ConvertScripts &&
csvRegex.IsMatch(x.FullPath))
.ToList();
var texturesToConvert = ark.Entries
.Where(x => op.ConvertTextures &&
textureRegex.IsMatch(x.FullPath))
.ToList();
var milosToInflate = ark.Entries
.Where(x => op.InflateMilos &&
!op.ExtractMilos &&
miloRegex.IsMatch(x.FullPath))
.ToList();
var milosToExtract = ark.Entries
.Where(x => op.ExtractMilos &&
miloRegex.IsMatch(x.FullPath))
.ToList();
var entriesToExtract = ark.Entries
.Where(x => op.ExtractAll)
.Except(scriptsToConvert)
.Except(texturesToConvert)
.Except(milosToInflate)
.ToList();
foreach (var arkEntry in entriesToExtract)
{
var filePath = ExtractEntry(ark, arkEntry, CombinePath(op.OutputPath, arkEntry.FullPath));
Console.WriteLine($"Wrote \"{filePath}\"");
}
// Create temp path
var tempDir = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, true);
}
foreach (var textureEntry in texturesToConvert)
{
using var arkEntryStream = ark.GetArkEntryFileStream(textureEntry);
var filePath = CombinePath(op.OutputPath, textureEntry.FullPath);
var pngPath = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(filePath)), Path.GetFileNameWithoutExtension(filePath) + ".png");
// Removes gen sub directory
if (genPathedFile.IsMatch(pngPath))
{
var match = genPathedFile.Match(pngPath);
pngPath = $"{match.Groups[1]}{match.Groups[4]}";
}
var info = new SystemInfo()
{
Version = 10,
Platform = Platform.PS2,
BigEndian = false
};
var serializer = new MiloSerializer(info);
var bitmap = serializer.ReadFromStream <HMXBitmap>(arkEntryStream);
bitmap.SaveAs(info, pngPath);
Console.WriteLine($"Wrote \"{pngPath}\"");
}
foreach (var miloEntry in milosToInflate)
{
var filePath = ExtractEntry(ark, miloEntry, CombinePath(op.OutputPath, miloEntry.FullPath));
// Inflate milo
var milo = MiloFile.ReadFromFile(filePath);
milo.Structure = BlockStructure.MILO_A;
milo.WriteToFile(filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
foreach (var miloEntry in milosToExtract)
{
var filePath = ExtractEntry(ark, miloEntry, CombinePath(op.OutputPath, miloEntry.FullPath));
var dirPath = Path.GetDirectoryName(filePath);
var tempPath = filePath + "_temp";
File.Move(filePath, tempPath, true);
var extPath = Path.Combine(
Path.GetDirectoryName(filePath),
Path.GetFileName(filePath));
var state = new AppState(dirPath);
state.ExtractMiloContents(
Path.GetFileName(tempPath),
extPath,
op.ConvertTextures);
// TODO: Refactor IDirectory and remove temp file write/delete
File.Delete(tempPath);
Console.WriteLine($"Wrote \"{extPath}\"");
}
foreach (var csvEntry in csvsToConvert)
{
var csvStream = ark.GetArkEntryFileStream(csvEntry);
var csv = CSVFile.FromForgeCSVStream(csvStream);
// Write to file
var csvPath = CombinePath(op.OutputPath, csvEntry.FullPath);
csvPath = platformExtRegex.Replace(csvPath, "");
csv.SaveToFileAsCSV(csvPath);
Console.WriteLine($"Wrote \"{csvPath}\"");
}
var successDtas = 0;
foreach (var scriptEntry in scriptsToConvert)
{
// Just extract file if dta script
if (dtaRegex.IsMatch(scriptEntry.FullPath))
{
var filePath = ExtractEntry(ark, scriptEntry, CombinePath(op.OutputPath, scriptEntry.FullPath));
Console.WriteLine($"Wrote \"{filePath}\"");
continue;
}
// Creates output path
var dtaPath = CombinePath(op.OutputPath, scriptEntry.FullPath);
dtaPath = !scriptForgeRegex.IsMatch(dtaPath)
? $"{dtaPath.Substring(0, dtaPath.Length - 1)}a" // Simply change b -> a
: scriptForgeRegex.Replace(dtaPath, "");
// Removes gen sub directory
if (genPathedFile.IsMatch(dtaPath))
{
var match = genPathedFile.Match(dtaPath);
dtaPath = $"{match.Groups[1]}{match.Groups[4]}";
}
var tempDtbPath = ExtractEntry(ark, scriptEntry, Path.Combine(tempDir, Path.GetRandomFileName()));
try
{
ScriptHelper.ConvertDtbToDta(tempDtbPath, tempDir, arkEncrypted, arkVersion, dtaPath, op.IndentSize);
Console.WriteLine($"Wrote \"{dtaPath}\"");
successDtas++;
}
catch (DTBParseException ex)
{
Console.WriteLine($"Unable to convert to script, skipping \'{scriptEntry.FullPath}\'");
if (File.Exists(dtaPath))
{
File.Delete(dtaPath);
}
}
catch (Exception ex)
{
}
}
if (scriptsToConvert.Count > 0)
{
Console.WriteLine($"Converted {successDtas} of {scriptsToConvert.Count} scripts");
}
// Clean up temp files
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, true);
}
}