public override void ReadFromStream(AwesomeReader ar, ISerializable data)
{
var env = data as Environ;
int version = ReadMagic(ar, data);
MiloSerializer.ReadFromStream(ar.BaseStream, env.Draw);
// Read lights
var lightCount = ar.ReadInt32();
env.Lights.Clear();
env.Lights.AddRange(RepeatFor(lightCount, () => ar.ReadString()));
env.AmbientColor = new Color4()
{
R = ar.ReadSingle(),
G = ar.ReadSingle(),
B = ar.ReadSingle(),
A = ar.ReadSingle()
};
env.FogStart = ar.ReadSingle();
env.FogEnd = ar.ReadSingle();
env.FogColor = new Color4()
{
R = ar.ReadSingle(),
G = ar.ReadSingle(),
B = ar.ReadSingle(),
A = ar.ReadSingle()
};
env.EnableFog = ar.ReadBoolean();
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var env = data as Environ;
// TODO: Add version check
var version = Magic();
aw.Write(version);
MiloSerializer.WriteToStream(aw.BaseStream, env.Draw);
// Write lights
aw.Write((int)env.Lights.Count);
env.Lights.ForEach(x => aw.Write((string)x));
// Write ambient color
aw.Write((float)env.AmbientColor.R);
aw.Write((float)env.AmbientColor.G);
aw.Write((float)env.AmbientColor.B);
aw.Write((float)env.AmbientColor.A);
// Write fog info
aw.Write((float)env.FogStart);
aw.Write((float)env.FogEnd);
aw.Write((float)env.FogColor.R);
aw.Write((float)env.FogColor.G);
aw.Write((float)env.FogColor.B);
aw.Write((float)env.FogColor.A);
aw.Write((bool)env.EnableFog);
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var cam = data as Cam;
// TODO: Add version check
var version = Magic();
aw.Write(version);
MiloSerializer.WriteToStream(aw.BaseStream, cam.Trans);
MiloSerializer.WriteToStream(aw.BaseStream, cam.Draw);
aw.Write((float)cam.NearPlane);
aw.Write((float)cam.FarPlane);
aw.Write((float)cam.FOV);
// Write screen area
aw.Write((float)cam.ScreenArea.X);
aw.Write((float)cam.ScreenArea.Y);
aw.Write((float)cam.ScreenArea.Width);
aw.Write((float)cam.ScreenArea.Height);
// Write z-range
aw.Write((float)cam.ZRange.X);
aw.Write((float)cam.ZRange.Y);
aw.Write((string)cam.TargetTexture);
}
public override void ReadFromStream(AwesomeReader ar, ISerializable data)
{
var cam = data as Cam;
int version = ReadMagic(ar, data);
MiloSerializer.ReadFromStream(ar.BaseStream, cam.Trans);
MiloSerializer.ReadFromStream(ar.BaseStream, cam.Draw);
cam.NearPlane = ar.ReadSingle();
cam.FarPlane = ar.ReadSingle();
cam.FOV = ar.ReadSingle();
// Read screen area
cam.ScreenArea = new Rectangle()
{
X = ar.ReadSingle(),
Y = ar.ReadSingle(),
Width = ar.ReadSingle(),
Height = ar.ReadSingle()
};
// Read z-range
cam.ZRange = new Vector2()
{
X = ar.ReadSingle(),
Y = ar.ReadSingle()
};
cam.TargetTexture = ar.ReadString();
}
public static byte[] WriteToBytes(this MiloSerializer serializer, ISerializable obj)
{
using (var ms = new MemoryStream())
{
serializer.WriteToStream(ms, obj);
return(ms.ToArray());
}
}
public static T ReadFromMiloObjectBytes <T>(this MiloSerializer serializer, MiloObjectBytes entry) where T : ISerializable, new()
{
using (var ms = new MemoryStream(entry.Data))
{
var obj = serializer.ReadFromStream <T>(ms);
(obj as MiloObject).Name = entry.Name;
return(obj);
}
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var view = data as View;
// TODO: Add version check
var version = Magic();
aw.Write(version);
MiloSerializer.WriteToStream(aw.BaseStream, view.Anim);
MiloSerializer.WriteToStream(aw.BaseStream, view.Trans);
MiloSerializer.WriteToStream(aw.BaseStream, view.Draw);
aw.Write((string)view.MainView);
aw.Write((float)view.LODHeight);
aw.Write((float)view.LODWidth);
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var tex = data as Tex;
// TODO: Add version check
var version = Magic();
aw.Write((int)version);
if (MiloSerializer.Info.Version >= 25)
{
aw.Write((int)1); // Definitely needed!
}
if (version >= 10)
{
aw.Write(new byte[9]);
}
aw.Write((int)tex.Width);
aw.Write((int)tex.Height);
aw.Write((int)tex.Bpp);
aw.Write(tex.ExternalPath);
aw.Write((float)tex.IndexF);
aw.Write((int)tex.Index);
aw.Write((bool)tex.UseExternal);
if (tex.Bitmap != null)
{
MiloSerializer.WriteToStream(aw.BaseStream, tex.Bitmap);
}
/*
* if (!tex.UseExternal && tex.Bitmap != null)
* {
* aw.Write(false);
* MiloSerializer.WriteToStream(aw.BaseStream, tex.Bitmap);
* }
* else
* {
* aw.Write(true);
* }*/
}
public override void ReadFromStream(AwesomeReader ar, ISerializable data)
{
var view = data as View;
int version = ReadMagic(ar, data);
var meta = ReadMeta(ar);
MiloSerializer.ReadFromStream(ar.BaseStream, view.Anim);
MiloSerializer.ReadFromStream(ar.BaseStream, view.Trans);
MiloSerializer.ReadFromStream(ar.BaseStream, view.Draw);
if (version >= 11)
{
// Read draw group
var drawableCount = ar.ReadInt32();
view.Draw.Drawables.Clear();
view.Draw.Drawables.AddRange(RepeatFor(drawableCount, () => ar.ReadString()));
}
view.MainView = ar.ReadString();
if (version == 11)
{
// Has less data at end of file
return;
}
// Ratio is usually 4:3
view.LODHeight = ar.ReadSingle();
view.LODWidth = ar.ReadSingle();
// 9 extra bytes for v14?
/*
* if (view.ScreenHeight > 0.0f && (view.ScreenWidth / view.ScreenHeight) != (4.0f / 3.0f))
* throw new Exception($"Aspect ratio should be {(4.0f / 3.0f):F2}, got {(view.ScreenWidth / view.ScreenHeight):F2}");
*/
}
public P9SongPrefSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public MatSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var dir = data as MiloObjectDir;
aw.Write((int)Magic());
// Sort entries using sort order defined in games
var sortedEntries = dir.Entries
.OrderBy(x => GetEntryTypeSortValue(x.Type))
.ToList();
if (Magic() >= 24)
{
// Write extra info
var dirEntry = dir.Extras["DirectoryEntry"] as MiloObject;
aw.Write((string)dirEntry.Type);
aw.Write((string)dirEntry.Name);
// Calculate hash + blob sizes
var hashCount = (sortedEntries.Count + 1) * 2;
var blobSize = sortedEntries.Select(x => x.Name.Length + 1).Sum() + (dirEntry.Name.Length + 1);
aw.Write((int)hashCount);
aw.Write((int)blobSize);
}
// Write entries
aw.Write((int)sortedEntries.Count);
foreach (var entry in sortedEntries)
{
// Used to preserve file name
var dirtyName = UnsanitizeFileName(entry.Name);
aw.Write((string)entry.Type);
aw.Write((string)dirtyName);
}
if (Magic() >= 24)
{
var dirEntryRaw = dir.Extras["DirectoryEntry"] as ISerializable;
if (Magic() == 25 &&
dir.Type == "ObjectDir" &&
dirEntryRaw is MiloObjectDirEntry dirEntry)
{
// Hack for project 9
aw.Write((int)dirEntry.Version);
aw.Write((int)dirEntry.SubVersion);
aw.Write((string)dirEntry.ProjectName);
// Write matrices
aw.Write((int)7);
foreach (var mat in Enumerable
.Range(0, 7)
.Select(x => Matrix4.Identity()))
{
aw.Write((float)mat.M11);
aw.Write((float)mat.M12);
aw.Write((float)mat.M13);
aw.Write((float)mat.M21);
aw.Write((float)mat.M22);
aw.Write((float)mat.M23);
aw.Write((float)mat.M31);
aw.Write((float)mat.M32);
aw.Write((float)mat.M33);
aw.Write((float)mat.M41);
aw.Write((float)mat.M42);
aw.Write((float)mat.M43);
}
// Constants? I hope
aw.Write((int)0);
aw.Write((bool)true);
aw.Write((int)0);
// Write imported milo paths
if (!(dirEntry.ImportedMiloPaths is null))
{
aw.Write((int)dirEntry.ImportedMiloPaths.Length);
foreach (var path in dirEntry.ImportedMiloPaths)
{
aw.Write((string)path);
}
}
else
{
aw.Write((int)0);
}
// Might be different depending on dir being root/nested
// Root: false, Nested: true
aw.Write((bool)(dirEntry.SubDirectories.Count <= 0)); // TODO: Use a better way to determine if nested
// Write sub directory names
aw.Write((int)dirEntry.SubDirectories.Count);
foreach (var subName in dirEntry
.SubDirectories
.Select(x => $"{x.Name}.milo")
.Reverse()) // Seems to be reverse order of serialization
{
aw.Write((string)subName);
}
// Write sub directory data
foreach (var subDir in dirEntry.SubDirectories)
{
WriteToStream(aw, subDir);
}
aw.BaseStream.Position += 13; // Zero'd bytes
}
else
{
MiloSerializer.WriteToStream(aw.BaseStream, dirEntryRaw);
}
aw.Write(ADDE_PADDING);
}
public MiloObjectDirSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public FontSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public override void ReadFromStream(AwesomeReader ar, ISerializable data)
{
var mesh = data as Mesh;
int version = ReadMagic(ar, data);
var meta = ReadMeta(ar);
MiloSerializer.ReadFromStream(ar.BaseStream, mesh.Trans);
MiloSerializer.ReadFromStream(ar.BaseStream, mesh.Draw);
mesh.Material = ar.ReadString();
mesh.MainMesh = ar.ReadString();
mesh.Unknown = ar.ReadInt32();
switch (mesh.Unknown)
{
case 0:
case 31:
case 33:
case 37:
case 63:
break;
default:
throw new Exception($"Unexpected number, got {mesh.Unknown}");
}
var num = ar.ReadInt32();
if (!(num == 0 || num == 1))
{
throw new Exception($"This should be 0 or 1, got {num}");
}
num = ar.ReadByte();
if (num != 0)
{
throw new Exception($"This should be 0, got {num}");
}
// Read vertices
var count = ar.ReadInt32();
if (version >= 36)
{
ar.BaseStream.Position += 9; // Skips unknown stuff
}
mesh.Vertices.Clear();
mesh.Vertices.AddRange(RepeatFor(count, () =>
{
var vertex = new Vertex3();
vertex.X = ar.ReadSingle();
vertex.Y = ar.ReadSingle();
vertex.Z = ar.ReadSingle();
if (version == 34)
{
ar.BaseStream.Position += 4; // Skip W for RB1
}
if (version < 35)
{
// Single precision
vertex.NormalX = ar.ReadSingle();
vertex.NormalY = ar.ReadSingle();
vertex.NormalZ = ar.ReadSingle();
if (version == 34)
{
ar.BaseStream.Position += 4; // Skip W for RB1
}
vertex.ColorR = ar.ReadSingle();
vertex.ColorG = ar.ReadSingle();
vertex.ColorB = ar.ReadSingle();
vertex.ColorA = ar.ReadSingle();
vertex.U = ar.ReadSingle();
vertex.V = ar.ReadSingle();
if (version == 34)
{
ar.BaseStream.Position += 24; // Skip unknown bytes for RB1
}
}
else
{
// Half precision
vertex.U = ar.ReadHalf();
vertex.V = ar.ReadHalf();
// Not sure what this value is but it's usually pretty high
ar.BaseStream.Position += 8;
// Skip reading normals for now
//vertex.NormalX = ar.ReadHalf();
//vertex.NormalY = ar.ReadHalf();
//vertex.NormalZ = ar.ReadHalf();
vertex.NormalX = 1.0f;
vertex.NormalY = 1.0f;
vertex.NormalZ = 1.0f;
vertex.ColorR = ar.ReadByte();
vertex.ColorG = ar.ReadByte();
vertex.ColorB = ar.ReadByte();
vertex.ColorA = ar.ReadByte();
// Skip unknown bytes
ar.BaseStream.Position += 8;
}
return(vertex);
}));
// Read face indicies
count = ar.ReadInt32();
mesh.Faces.Clear();
mesh.Faces.AddRange(RepeatFor(count, () => new Face()
{
V1 = ar.ReadUInt16(),
V2 = ar.ReadUInt16(),
V3 = ar.ReadUInt16(),
}));
// Read groups
count = ar.ReadInt32();
var groupSizes = RepeatFor(count, () => ar.ReadByte()).ToArray();
if (groupSizes.Select(x => (int)x).Sum() != mesh.Faces.Count)
{
throw new Exception("Sum should equal count of faces");
}
var charCount = ar.ReadInt32();
ar.BaseStream.Position -= 4;
// Read bones
mesh.Bones.Clear();
if (charCount > 0)
{
if (version >= 36)
{
// Uses variable length bone count
ar.BaseStream.Position += 4;
mesh.Bones
.AddRange(RepeatFor(charCount, () => new Bone()
{
Name = ar.ReadString(),
Mat = ReadMatrix(ar)
}));
}
else
{
// Uses constant length bone count
const int boneCount = 4; // Always 4?
var boneNames = RepeatFor(boneCount, () => ar.ReadString()).ToArray(); // Either 3 or none (Last one is always empty?)
var boneMats = RepeatFor(boneCount, () => ReadMatrix(ar)).ToArray();
for (int i = 0; i < boneCount; i++)
{
mesh.Bones.Add(new Bone()
{
Name = boneNames[i],
Mat = boneMats[i]
});
}
}
}
else
{
// Skips zero
ar.BaseStream.Position += 4;
}
if (version == 36)
{
ar.BaseStream.Position += 1;
}
else if (version >= 37)
{
ar.BaseStream.Position += 2;
}
mesh.Groups.Clear();
if (count <= 0 || groupSizes[0] <= 0 || ar.BaseStream.Length == ar.BaseStream.Position)
{
mesh.Groups.AddRange(groupSizes
.Select(x => new FaceGroup()
{
Size = x,
Sections = new List <int>(),
VertexIndicies = new List <int>()
}));
return;
}
// Read face groups
mesh.Groups.AddRange(Enumerable.Range(0, count).Select(x =>
{
var sectionCount = ar.ReadInt32();
var vertCount = ar.ReadInt32();
return(new FaceGroup()
{
Size = groupSizes[x],
Sections = Enumerable.Range(0, sectionCount)
.Select(y => ar.ReadInt32())
.ToList(),
VertexIndicies = Enumerable.Range(0, vertCount)
.Select(y => (int)ar.ReadUInt16())
.ToList(),
});
}));
}
public void Parse(Ark2DirOptions op)
{
var scriptRegex = new Regex("(?i).((dtb)|(dta)|(([A-Z]+)(_dta_)([A-Z0-9]+)))$");
var scriptForgeRegex = new Regex("(?i)(_dta_)([A-Z0-9]+)$");
var csvRegex = new Regex("(?i).csv_([A-Z0-9]+)$");
var dtaRegex = new Regex("(?i).dta$");
var textureRegex = new Regex("(?i).((bmp)|(png))(_[A-Z0-9]+)$");
var miloRegex = new Regex("(?i).((gh)|(milo)|(rnd))(_[A-Z0-9]+)?$");
var genPathedFile = new Regex(@"(?i)(([^\/\\]+[\/\\])*)(gen[\/\\])([^\/\\]+)$");
var platformExtRegex = new Regex(@"(?i)_([A-Z0-9]+)$");
Archive ark;
int arkVersion;
bool arkEncrypted;
if (Directory.Exists(op.InputPath))
{
// Open as directory
ark = ArkFileSystem.FromDirectory(op.InputPath);
// TODO: Get from args probably
arkVersion = 10;
arkEncrypted = true;
}
else
{
// Open as ark
var arkFile = ArkFile.FromFile(op.InputPath);
arkVersion = (int)arkFile.Version;
arkEncrypted = arkFile.Encrypted;
ark = arkFile;
}
var scriptsToConvert = ark.Entries
.Where(x => op.ConvertScripts &&
arkVersion >= 3 && // Amp dtbs not supported right now
scriptRegex.IsMatch(x.FullPath))
.ToList();
var csvsToConvert = ark.Entries
.Where(x => op.ConvertScripts &&
csvRegex.IsMatch(x.FullPath))
.ToList();
var texturesToConvert = ark.Entries
.Where(x => op.ConvertTextures &&
textureRegex.IsMatch(x.FullPath))
.ToList();
var milosToInflate = ark.Entries
.Where(x => op.InflateMilos &&
!op.ExtractMilos &&
miloRegex.IsMatch(x.FullPath))
.ToList();
var milosToExtract = ark.Entries
.Where(x => op.ExtractMilos &&
miloRegex.IsMatch(x.FullPath))
.ToList();
var entriesToExtract = ark.Entries
.Where(x => op.ExtractAll)
.Except(scriptsToConvert)
.Except(texturesToConvert)
.Except(milosToInflate)
.ToList();
foreach (var arkEntry in entriesToExtract)
{
var filePath = ExtractEntry(ark, arkEntry, CombinePath(op.OutputPath, arkEntry.FullPath));
Console.WriteLine($"Wrote \"{filePath}\"");
}
// Create temp path
var tempDir = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, true);
}
foreach (var textureEntry in texturesToConvert)
{
using var arkEntryStream = ark.GetArkEntryFileStream(textureEntry);
var filePath = CombinePath(op.OutputPath, textureEntry.FullPath);
var pngPath = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(filePath)), Path.GetFileNameWithoutExtension(filePath) + ".png");
// Removes gen sub directory
if (genPathedFile.IsMatch(pngPath))
{
var match = genPathedFile.Match(pngPath);
pngPath = $"{match.Groups[1]}{match.Groups[4]}";
}
var info = new SystemInfo()
{
Version = 10,
Platform = Platform.PS2,
BigEndian = false
};
var serializer = new MiloSerializer(info);
var bitmap = serializer.ReadFromStream <HMXBitmap>(arkEntryStream);
bitmap.SaveAs(info, pngPath);
Console.WriteLine($"Wrote \"{pngPath}\"");
}
foreach (var miloEntry in milosToInflate)
{
var filePath = ExtractEntry(ark, miloEntry, CombinePath(op.OutputPath, miloEntry.FullPath));
// Inflate milo
var milo = MiloFile.ReadFromFile(filePath);
milo.Structure = BlockStructure.MILO_A;
milo.WriteToFile(filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
foreach (var miloEntry in milosToExtract)
{
var filePath = ExtractEntry(ark, miloEntry, CombinePath(op.OutputPath, miloEntry.FullPath));
var dirPath = Path.GetDirectoryName(filePath);
var tempPath = filePath + "_temp";
File.Move(filePath, tempPath, true);
var extPath = Path.Combine(
Path.GetDirectoryName(filePath),
Path.GetFileName(filePath));
var state = new AppState(dirPath);
state.ExtractMiloContents(
Path.GetFileName(tempPath),
extPath,
op.ConvertTextures);
// TODO: Refactor IDirectory and remove temp file write/delete
File.Delete(tempPath);
Console.WriteLine($"Wrote \"{extPath}\"");
}
foreach (var csvEntry in csvsToConvert)
{
var csvStream = ark.GetArkEntryFileStream(csvEntry);
var csv = CSVFile.FromForgeCSVStream(csvStream);
// Write to file
var csvPath = CombinePath(op.OutputPath, csvEntry.FullPath);
csvPath = platformExtRegex.Replace(csvPath, "");
csv.SaveToFileAsCSV(csvPath);
Console.WriteLine($"Wrote \"{csvPath}\"");
}
var successDtas = 0;
foreach (var scriptEntry in scriptsToConvert)
{
// Just extract file if dta script
if (dtaRegex.IsMatch(scriptEntry.FullPath))
{
var filePath = ExtractEntry(ark, scriptEntry, CombinePath(op.OutputPath, scriptEntry.FullPath));
Console.WriteLine($"Wrote \"{filePath}\"");
continue;
}
// Creates output path
var dtaPath = CombinePath(op.OutputPath, scriptEntry.FullPath);
dtaPath = !scriptForgeRegex.IsMatch(dtaPath)
? $"{dtaPath.Substring(0, dtaPath.Length - 1)}a" // Simply change b -> a
: scriptForgeRegex.Replace(dtaPath, "");
// Removes gen sub directory
if (genPathedFile.IsMatch(dtaPath))
{
var match = genPathedFile.Match(dtaPath);
dtaPath = $"{match.Groups[1]}{match.Groups[4]}";
}
var tempDtbPath = ExtractEntry(ark, scriptEntry, Path.Combine(tempDir, Path.GetRandomFileName()));
try
{
ScriptHelper.ConvertDtbToDta(tempDtbPath, tempDir, arkEncrypted, arkVersion, dtaPath, op.IndentSize);
Console.WriteLine($"Wrote \"{dtaPath}\"");
successDtas++;
}
catch (DTBParseException ex)
{
Console.WriteLine($"Unable to convert to script, skipping \'{scriptEntry.FullPath}\'");
if (File.Exists(dtaPath))
{
File.Delete(dtaPath);
}
}
catch (Exception ex)
{
}
}
if (scriptsToConvert.Count > 0)
{
Console.WriteLine($"Converted {successDtas} of {scriptsToConvert.Count} scripts");
}
// Clean up temp files
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, true);
}
}
public TexSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public DrawSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var dir = data as MiloObjectDir;
aw.Write((int)Magic());
if (Magic() >= 24)
{
// Write extra info
var dirEntry = dir.Extras["DirectoryEntry"] as MiloObject;
aw.Write((string)dirEntry.Type);
aw.Write((string)dirEntry.Name);
/*
* var entryNameLengths = dir.Entries.Select(x => ((string)x.Name).Length).Sum();
* aw.Write(0); // Unknown
* aw.Write(1 + ((string)dirEntry.Name).Length + dir.Entries.Count + entryNameLengths);
*/
if (dir.Extras.TryGetValue("Num1", out var num1))
{
aw.Write((int)num1);
}
if (dir.Extras.TryGetValue("Num2", out var num2))
{
aw.Write((int)num2);
}
aw.Write((int)dir.Entries.Count);
}
foreach (var entry in dir.Entries)
{
aw.Write((string)entry.Type);
aw.Write((string)entry.Name);
}
if (Magic() >= 24)
{
var dirEntryRaw = dir.Extras["DirectoryEntry"] as ISerializable;
if (Magic() == 25 &&
dir.Type == "ObjectDir" &&
dirEntryRaw is MiloObjectDirEntry dirEntry)
{
// Hack for project 9
aw.Write((int)dirEntry.Version);
aw.Write((int)dirEntry.SubVersion);
aw.Write((string)dirEntry.ProjectName);
// Write matrices
aw.Write((int)7);
foreach (var mat in Enumerable
.Range(0, 7)
.Select(x => Matrix4.Identity()))
{
aw.Write((float)mat.M11);
aw.Write((float)mat.M12);
aw.Write((float)mat.M13);
aw.Write((float)mat.M21);
aw.Write((float)mat.M22);
aw.Write((float)mat.M23);
aw.Write((float)mat.M31);
aw.Write((float)mat.M32);
aw.Write((float)mat.M33);
aw.Write((float)mat.M41);
aw.Write((float)mat.M42);
aw.Write((float)mat.M43);
}
// Constants? I hope
aw.Write((int)0);
aw.Write((bool)true);
aw.Write((int)0);
// Write imported milo paths
if (!(dirEntry.ImportedMiloPaths is null))
{
aw.Write((int)dirEntry.ImportedMiloPaths.Length);
foreach (var path in dirEntry.ImportedMiloPaths)
{
aw.Write((string)path);
}
}
else
{
aw.Write((int)0);
}
// Might be different depending on dir being root/nested
// Root: false, Nested: true
aw.Write((bool)(dirEntry.SubDirectories.Count <= 0)); // TODO: Use a better way to determine if nested
// Write sub directory names
aw.Write((int)dirEntry.SubDirectories.Count);
foreach (var subName in dirEntry
.SubDirectories
.Select(x => $"{x.Name}.milo")
.Reverse()) // Seems to be reverse order of serialization
{
aw.Write((string)subName);
}
// Write sub directory data
foreach (var subDir in dirEntry.SubDirectories)
{
WriteToStream(aw, subDir);
}
aw.BaseStream.Position += 13; // Zero'd bytes
}
else
{
MiloSerializer.WriteToStream(aw.BaseStream, dirEntryRaw);
}
aw.Write(ADDE_PADDING);
}
public ViewSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public TransSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public EnvironSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public HMXBitmapSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public AnimSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public override void ReadFromStream(AwesomeReader ar, ISerializable data)
{
var tex = data as Tex;
int version = ReadMagic(ar, data);
// Skips zeros
if (version >= 10 && MiloSerializer.Info.Version == 24)
{
ar.BaseStream.Position += 9; // GH2 PS2
}
else if (version >= 10)
{
// GH2 360 (13 bytes when no script)
ar.BaseStream.Position += 4; // Revision number? Usually 1 or 2
tex.ScriptName = ar.ReadString(); // Script name?
var hasDtb = ar.ReadBoolean();
if (hasDtb)
{
ar.BaseStream.Position -= 1;
tex.Script = DTBFile.FromStream(ar, DTBEncoding.Classic);
}
// Extra meta (comment?)
var meta = ar.ReadString();
if (version >= 11)
{
ar.BaseStream.Position += 1; // Not sure why it has an extra byte
}
}
tex.Width = ar.ReadInt32();
tex.Height = ar.ReadInt32();
tex.Bpp = ar.ReadInt32();
tex.ExternalPath = ar.ReadString();
tex.IndexF = ar.ReadSingle();
if (tex.IndexF < -13.0f || (tex.IndexF > 0.0f && tex.IndexF != 666.0f))
{
throw new NotSupportedException($"Expected number between -13.0 <-> 0.0, got {tex.IndexF}");
}
tex.Index = ar.ReadInt32();
switch (tex.Index)
{
case 1:
case 2:
case 4:
case 8: // krpa
case 34:
break;
default:
throw new NotSupportedException($"Unexpected number, got {tex.Index}");
}
tex.UseExternal = ar.ReadBoolean();
tex.Bitmap = null;
if (ar.BaseStream.Position == ar.BaseStream.Length)
{
return;
}
tex.Bitmap = MiloSerializer.ReadFromStream <HMXBitmap>(ar.BaseStream);
}
public IActionResult ScanArkPost([FromBody] ScanRequest request)
{
_miloContext.Database.EnsureCreated();
var sw = Stopwatch.StartNew();
// Updates games (arks)
var game = _miloContext.Arks.FirstOrDefault(x => x.Title == request.GameTitle &&
x.Platform == request.Platform &&
x.Region == request.Region);
if (game == null)
{
// Create game
game = new Data.MiloEntities.Ark()
{
Title = request.GameTitle,
Platform = request.Platform,
Region = request.Region
};
_miloContext.Arks.Add(game);
_miloContext.SaveChanges();
}
var ark = ArkFile.FromFile(request.InputPath);
game.ArkVersion = (int)ark.Version;
var miloEntries = new List <Data.MiloEntities.ArkEntry>();
var totalMiloEntries = 0;
// Updates ark entries
foreach (var arkEntry in ark.Entries)
{
var entry = arkEntry as OffsetArkEntry;
var contextEntry = _miloContext.ArkEntries.FirstOrDefault(x => x.Ark == game && x.Path == entry.FullPath);
if (contextEntry == null)
{
contextEntry = new Data.MiloEntities.ArkEntry()
{
Ark = game,
Path = entry.FullPath
};
_miloContext.ArkEntries.Add(contextEntry);
_miloContext.SaveChanges();
}
contextEntry.Part = entry.Part;
contextEntry.Offset = entry.Offset;
contextEntry.Size = (int)entry.Size;
contextEntry.InflatedSize = (int)entry.InflatedSize;
if (_miloRegex.IsMatch(contextEntry.Path))
{
miloEntries.Add(contextEntry);
}
_miloContext.Update(contextEntry);
}
// Updates milos
foreach (var miloEntry in miloEntries)
{
var arkEntry = ark.Entries.First(x => x.FullPath == miloEntry.Path);
var mf = MiloFile.ReadFromStream(ark.GetArkEntryFileStream(arkEntry));
var serializer = new MiloSerializer(new SystemInfo()
{
BigEndian = mf.BigEndian, Version = mf.Version
});
MiloObjectDir milo;
using (var ms = new MemoryStream(mf.Data))
{
milo = serializer.ReadFromStream <MiloObjectDir>(ms);
}
totalMiloEntries += milo.Entries.Count;
var contextEntry = _miloContext.Milos.FirstOrDefault(x => x.ArkEntry == miloEntry);
if (contextEntry == null)
{
contextEntry = new Data.MiloEntities.Milo()
{
ArkEntry = miloEntry
};
_miloContext.Milos.Add(contextEntry);
_miloContext.SaveChanges();
}
contextEntry.Version = mf.Version;
contextEntry.TotalSize = mf.Data.Length;
contextEntry.Name = milo.Name ?? "";
contextEntry.Type = milo.Type ?? "";
var dirEntry = milo.Entries
.Where(x => ((string)x.Type).EndsWith("Dir") && x is MiloObjectBytes)
.Select(x => x as MiloObjectBytes)
.FirstOrDefault();
if (dirEntry != null)
{
contextEntry.Size = dirEntry.Data.Length;
contextEntry.Magic = dirEntry.GetMagic();
}
else
{
contextEntry.Size = -1;
contextEntry.Magic = -1;
}
// Updates milo entries
foreach (var mEntry in milo.Entries.Where(x => x is MiloObjectBytes && x != dirEntry).Select(y => y as MiloObjectBytes))
{
var contextMEntry = _miloContext.MiloEntries.FirstOrDefault(x => x.Milo == contextEntry && x.Name == mEntry.Name && x.Type == mEntry.Type);
if (contextMEntry == null)
{
contextMEntry = new Data.MiloEntities.MiloEntry()
{
Milo = contextEntry
};
_miloContext.MiloEntries.Add(contextMEntry);
_miloContext.SaveChanges();
}
contextMEntry.Name = mEntry.Name ?? "";
contextMEntry.Type = mEntry.Type ?? "";
contextMEntry.Size = mEntry.Data.Length;
contextMEntry.Magic = mEntry.GetMagic();
_miloContext.Update(contextMEntry);
}
_miloContext.Update(contextEntry);
}
_miloContext.SaveChanges();
sw.Stop();
return(Ok(new ScanResult()
{
TotalArkEntries = ark.Entries.Count,
TotalMilos = miloEntries.Count,
TotalMiloEntries = totalMiloEntries,
TimeElapsed = sw.ElapsedMilliseconds
}));
}
public MeshSerializer(MiloSerializer miloSerializer) : base(miloSerializer)
{
}
public IActionResult TestSerialization([FromBody] ScanRequest request, bool testSerialize)
{
if (!Directory.Exists(request.InputPath))
{
return(BadRequest($"Directory \"{request.InputPath}\" does not exist!"));
}
var miloEntryRegex = new Regex(@"([^\\]*([.](rnd)|(gh)))\\([^\\]+)\\([^\\]+)$", RegexOptions.IgnoreCase);
var miloEntries = Directory.GetFiles(request.InputPath, "*", SearchOption.AllDirectories)
.Where(x => miloEntryRegex.IsMatch(x))
.Select(y =>
{
var match = miloEntryRegex.Match(y);
var miloArchive = match.Groups[1].Value;
var miloEntryType = match.Groups[5].Value;
var miloEntryName = match.Groups[6].Value;
return(new
{
FullPath = y,
MiloArchive = miloArchive,
MiloEntryType = miloEntryType,
MiloEntryName = miloEntryName
});
})
.ToArray();
//var groupedEntries = miloEntries.GroupBy(x => x.MiloEntryType).ToDictionary(g => g.Key, g => g.ToList());
MiloSerializer serializer = new MiloSerializer(new SystemInfo()
{
Version = 10, Platform = Platform.PS2, BigEndian = false
});
var supportedTypes = new [] { "Cam", "Environ", "Mat", "Mesh", "Tex", "View" };
var results = miloEntries
.Where(w => supportedTypes.Contains(w.MiloEntryType))
.OrderBy(x => x.MiloEntryType)
.ThenBy(y => y.FullPath)
.Select(z =>
{
ISerializable data = null;
string message = "";
bool converted = false;
bool perfectSerialize = true;
try
{
switch (z.MiloEntryType)
{
case "Cam":
data = serializer.ReadFromFile <Cam>(z.FullPath);
break;
case "Environ":
data = serializer.ReadFromFile <Environ>(z.FullPath);
break;
case "Mat":
data = serializer.ReadFromFile <Mat>(z.FullPath);
break;
case "Mesh":
data = serializer.ReadFromFile <Mesh>(z.FullPath);
break;
case "Tex":
data = serializer.ReadFromFile <Tex>(z.FullPath);
break;
case "View":
data = serializer.ReadFromFile <View>(z.FullPath);
break;
default:
throw new Exception("Not Supported");
}
(data as IMiloObject).Name = z.MiloEntryName;
converted = true;
}
catch (Exception ex)
{
var trace = new StackTrace(ex, true);
var frame = trace.GetFrame(0);
var name = frame.GetMethod().ReflectedType.Name;
message = $"{name}: {ex.Message}";
}
if (testSerialize)
{
try
{
byte[] bytes;
using (var ms = new MemoryStream())
{
serializer.WriteToStream(ms, data);
bytes = ms.ToArray();
}
var origBytes = System.IO.File.ReadAllBytes(z.FullPath);
if (bytes.Length != origBytes.Length)
{
throw new Exception("Byte count doesn't match");
}
for (int i = 0; i < bytes.Length; i++)
{
if (bytes[i] != origBytes[i])
{
throw new Exception("Bytes don't match");
}
}
}
catch (Exception ex)
{
perfectSerialize = false;
}
}
return(new { Entry = z, Data = data, Message = message, Converted = converted, Serialized = perfectSerialize });
}).ToList();
/*
* var textures = groupedEntries["Tex"]
* .Select(x => serializer.ReadFromFile<Tex>(x.FullPath))
* .ToList();
*
* var views = groupedEntries["View"]
* .Select(x => serializer.ReadFromFile<View>(x.FullPath))
* .ToList();
*
* var meshes = groupedEntries["Mesh"]
* .Select(x => serializer.ReadFromFile<Mesh>(x.FullPath))
* .ToList();
*/
return(Ok(new
{
TotalCoverage = results.Count(x => x.Converted) / (double)results.Count,
TotalScanned = results.Count,
ByType = results
.GroupBy(x => x.Entry.MiloEntryType)
.ToDictionary(g => g.Key, g => new
{
Coverage = g.Count(x => x.Converted) / (double)g.Count(),
Scanned = g.Count(),
//Converted = g
// .Where(x => x.Converted)
// .Select(x => new
// {
// x.Entry.FullPath
// }),
NotConverted = g
.Where(x => !x.Converted)
.Select(x => new
{
x.Entry.FullPath,
x.Message
})
}),
NotSerialized = results
.Where(x => !x.Serialized)
.Select(y => y.Entry.FullPath)
//Converted = results
// .Where(x => x.Converted)
// .Select(x => new
// {
// x.Entry.FullPath
// }),
//NotConverted = results
// .Where(x => !x.Converted)
// .Select(x => new
// {
// x.Entry.FullPath,
// x.Message
// })
}));
}
public override void WriteToStream(AwesomeWriter aw, ISerializable data)
{
var mesh = data as Mesh;
// TODO: Add version check
var version = Magic();
aw.Write(version);
MiloSerializer.WriteToStream(aw.BaseStream, mesh.Trans);
MiloSerializer.WriteToStream(aw.BaseStream, mesh.Draw);
aw.Write((string)mesh.Material);
aw.Write((string)mesh.MainMesh);
aw.Write((int)mesh.Unknown);
aw.Write((int)1);
aw.Write((byte)0);
// Write vertices
aw.Write((int)mesh.Vertices.Count);
mesh.Vertices.ForEach(x =>
{
// TODO: Add switch statement for milo versions
aw.Write((float)x.X);
aw.Write((float)x.Y);
aw.Write((float)x.Z);
aw.Write((float)x.NormalX);
aw.Write((float)x.NormalY);
aw.Write((float)x.NormalZ);
aw.Write((float)x.ColorR);
aw.Write((float)x.ColorG);
aw.Write((float)x.ColorB);
aw.Write((float)x.ColorA);
aw.Write((float)x.U);
aw.Write((float)x.V);
});
// Write faces
aw.Write((int)mesh.Faces.Count);
mesh.Faces.ForEach(x =>
{
aw.Write((ushort)x.V1);
aw.Write((ushort)x.V2);
aw.Write((ushort)x.V3);
});
// Write group sizes
aw.Write((int)mesh.Groups.Count);
mesh.Groups.ForEach(x => aw.Write((byte)x.Size));
const int boneCount = 4; // Always 4?
var bones = mesh.Bones
.Take(boneCount)
.ToList();
// Write bones
if (bones.Count > 0)
{
bones.ForEach(x => aw.Write((string)x.Name));
RepeatFor(boneCount - bones.Count, () => aw.Write((string)""));
bones.ForEach(x => WriteMatrix(x.Mat, aw));
RepeatFor(boneCount - bones.Count, () => WriteMatrix(Matrix4.Identity(), aw));
}
else
{
aw.Write((int)0);
}
if (mesh.Groups.Sum(x => x.Sections.Count) == 0 &&
mesh.Groups.Sum(x => x.VertexIndicies.Count) == 0)
{
return;
}
// Write group sections + vertex indices
mesh.Groups.ForEach(x =>
{
aw.Write((int)x.Sections.Count);
aw.Write((int)x.VertexIndicies.Count);
x.Sections.ForEach(y => aw.Write((int)y));
x.VertexIndicies.ForEach(y => aw.Write((ushort)y));
});
}
public IActionResult ExtractFilesFromArk([FromBody] ScanRequest request, bool extractMilos, bool extractDTAs, bool convertTextures)
{
Archive ark;
if (!System.IO.File.Exists(request.InputPath))
{
// Open as directory if available
if (Directory.Exists(request.InputPath))
{
ark = ArkFileSystem.FromDirectory(request.InputPath);
}
else
{
return(BadRequest($"File \"{request.InputPath}\" does not exist!"));
}
}
else
{
// Open as archive
ark = ArkFile.FromFile(request.InputPath);
}
if (request.OutputPath == null)
{
return(BadRequest($"Output directory cannot be null!"));
}
string CombinePath(string basePath, string path)
{
// Consistent slash
basePath = (request.OutputPath ?? "").Replace("/", "\\");
path = (path ?? "").Replace("/", "\\");
Regex dotRegex = new Regex(@"[.]+[\\]");
if (dotRegex.IsMatch(path))
{
// Replaces dotdot path
path = dotRegex.Replace(path, x => $"({x.Value.Substring(0, x.Value.Length - 1)})\\");
}
return(Path.Combine(basePath, path));
}
string GetNonGenPath(string path)
{
// Consistent slash
path = (path ?? "").Replace("/", "\\");
Regex genRegex = new Regex(@"gen\\[^\\]+$", RegexOptions.IgnoreCase);
Regex platformRegex = new Regex(@"_[^_]+$", RegexOptions.IgnoreCase); // TODO: Revisit for Forge extensions
if (genRegex.IsMatch(path))
{
var splitPath = path.Split('\\');
var dir = string.Join("\\", splitPath.SkipLast(2));
var file = platformRegex.Replace(splitPath.Last(), "");
path = $"{dir}\\{file}";
}
return(path);
}
void SaveAsFile(MiloObjectBytes miloEntry, string basePath)
{
var fileName = SanitizeFileName(miloEntry.Name);
var filePath = basePath = Path.Combine(basePath, miloEntry.Type, fileName);
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
System.IO.File.WriteAllBytes(filePath, miloEntry.Data);
Console.WriteLine($"Wrote \"{filePath}\"");
}
// Extract everything
if (!extractDTAs && !extractMilos && !convertTextures)
{
foreach (var arkEntry in ark.Entries)
{
var filePath = CombinePath(request.OutputPath, arkEntry.FullPath);
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
using (var fs = System.IO.File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Write))
{
using (var stream = ark.GetArkEntryFileStream(arkEntry))
{
stream.CopyTo(fs);
}
}
Console.WriteLine($"Wrote \"{filePath}\"");
}
return(Ok());
}
string SanitizeFileName(string fileName)
{
// Sanitize file name
var invalidChars = new Regex($"[{new string(Path.GetInvalidFileNameChars())}]", RegexOptions.IgnoreCase);
return(invalidChars.Replace(fileName, ""));
}
void ProcessMiloArkEntry(ArkEntry miloArkEntry)
{
var filePath = CombinePath(request.OutputPath, GetNonGenPath(miloArkEntry.FullPath));
using (var stream = ark.GetArkEntryFileStream(miloArkEntry))
{
var milo = MiloFile.ReadFromStream(stream);
var miloSerializer = new MiloSerializer(new SystemInfo()
{
Version = milo.Version,
BigEndian = milo.BigEndian,
Platform = Enum.Parse <Platform>(request.Platform)
});
var miloDir = new MiloObjectDir();
using (var ms = new MemoryStream(milo.Data))
{
miloSerializer.ReadFromStream(ms, miloDir);
}
if (convertTextures)
{
var textureEntries = miloDir.Entries
.Where(x => x.Type == "Tex")
.Select(x => x is Tex ? x as Tex : miloSerializer.ReadFromMiloObjectBytes <Tex>(x as MiloObjectBytes))
.Where(x => x.Bitmap != null && x.Bitmap.RawData?.Length > 0)
.ToList();
if (textureEntries.Count <= 0)
{
return;
}
foreach (var texEntry in textureEntries)
{
var entryName = Path.GetFileNameWithoutExtension(SanitizeFileName(texEntry.Name));
var pngName = $"{entryName}.png";
var pngPath = Path.Combine(filePath, texEntry.Type, pngName);
texEntry.Bitmap.SaveAs(miloSerializer.Info, pngPath);
Console.WriteLine($"Wrote \"{pngPath}\"");
// Write DTA script to file
var scriptName = texEntry?.ScriptName ?? "";
if (texEntry.Script != null)
{
var dtaName = (string.IsNullOrEmpty(scriptName))
? $"{entryName}.dta"
: $"{entryName}_{scriptName}.dta";
var dtaPath = Path.Combine(filePath, texEntry.Type, dtaName);
var parent = new ParentItem(ParentType.Default);
foreach (var item in texEntry.Script.Items)
{
parent.Add(item);
}
texEntry.Script.Items.Clear();
texEntry.Script.Items.Add(parent);
System.IO.File.WriteAllText(dtaPath, texEntry.Script.ToString());
Console.WriteLine($"Wrote \"{dtaPath}\"");
}
}
}
if (extractMilos)
{
if (miloDir.Entries.Count <= 0)
{
return;
}
if (!Directory.Exists(Path.GetDirectoryName(filePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
// Saves milo entries
miloDir.Entries.ForEach(x =>
{
SaveAsFile(x as MiloObjectBytes, filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
});
if (miloDir.Extras.ContainsKey("DirectoryEntry"))
{
SaveAsFile(miloDir.Extras["DirectoryEntry"] as MiloObjectBytes, filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
}
}
}
void ProcessBitmapArkEntry(ArkEntry bitmapArkEntry)
{
var arkPath = GetNonGenPath(bitmapArkEntry.FullPath);
var filePath = CombinePath(request.OutputPath, $"{Path.GetDirectoryName(arkPath)}\\{Path.GetFileNameWithoutExtension(bitmapArkEntry.FileName)}.png");
using (var stream = ark.GetArkEntryFileStream(bitmapArkEntry))
{
var miloSerializer = new MiloSerializer(new SystemInfo()
{
Version = 25, // 10 = gh1, 24 = gh2
BigEndian = false, // Even on PPC it's LE
Platform = Enum.Parse <Platform>(request.Platform)
});
var bitmap = miloSerializer.ReadFromStream <HMXBitmap>(stream);
bitmap.SaveAs(miloSerializer.Info, filePath);
Console.WriteLine($"Wrote \"{filePath}\"");
}
}
// Extract milos
if (extractMilos || convertTextures)
{
var miloEntries = ark.Entries.Where(x => _miloRegex.IsMatch(x.FullPath));
Parallel.ForEach(miloEntries, (entry) =>
{
ProcessMiloArkEntry(entry);
});
//foreach (var entry in miloEntries) ProcessMiloArkEntry(entry);
}
// Convert textures
if (convertTextures)
{
var bitmapArkEntries = ark.Entries.Where(x => _bitmapRegex.IsMatch(x.FullPath));
Parallel.ForEach(bitmapArkEntries, (entry) =>
{
ProcessBitmapArkEntry(entry);
});
//foreach (var entry in bitmapArkEntries) ProcessBitmapArkEntry(entry);
}
return(Ok());
}
