public static Compression.IHyoutaArchiveCompressionInfo?Deserialize(DuplicatableStream stream, long maxBytes, EndianUtils.Endianness endian)
{
if (maxBytes < 8)
{
stream.DiscardBytes(maxBytes);
return(null);
}
ulong identifier = stream.ReadUInt64(EndianUtils.Endianness.BigEndian);
switch (identifier)
{
case 0:
// archive has compression, but this file is not compressed
stream.DiscardBytes(maxBytes - 8);
return(null);
case DeflateSharpCompressionInfo.Identifier:
return(DeflateSharpCompressionInfo.Deserialize(stream, maxBytes - 8, endian));
default:
Console.WriteLine("Unknown compression type: " + identifier.ToString("x16"));
stream.DiscardBytes(maxBytes - 8);
return(null);
}
}
public EndianUtils.Endianness Endian; // not actually a field in the header, at least not here -- might be the first byte?
public NubHeader(DuplicatableStream stream, EndianUtils.Endianness?endian)
{
Magic = stream.ReadUInt64(EndianUtils.Endianness.LittleEndian);
EndianUtils.Endianness suspectedEndian;
if (Magic == 0x10200)
{
suspectedEndian = EndianUtils.Endianness.BigEndian;
}
else if (Magic == 0x10201)
{
suspectedEndian = EndianUtils.Endianness.LittleEndian;
}
else
{
throw new Exception("unexpected magic in NUB");
}
EndianUtils.Endianness e = endian ?? suspectedEndian;
Fileid = stream.ReadUInt32(e); // or something like that? seems unique per archive in each game
EntryCount = stream.ReadUInt32(e);
StartOfFiles = stream.ReadUInt32(e);
FilesSize = stream.ReadUInt32(e);
StartOfEntries = stream.ReadUInt32(e);
StartOfHeaders = stream.ReadUInt32(e);
Endian = e;
}
private static string?ReadString(DuplicatableStream s, uint maxBytes, EndianUtils.Endianness e)
{
if (maxBytes < 8)
{
// can't be a valid string
s.DiscardBytes(maxBytes);
return(null);
}
ulong rawlength = s.ReadUInt64(e);
ulong length = (rawlength & 0x7ffffffffffffffful);
bool hasOffset = (rawlength & 0x8000000000000000ul) > 0;
if (hasOffset)
{
// format is 8 bytes length, then 8 bytes position of string in data
if (maxBytes < 16)
{
// can't be valid
s.DiscardBytes(maxBytes - 8);
return(null);
}
ulong offset = s.ReadUInt64(e);
long p = s.Position + (maxBytes - 16);
s.Position = (long)offset;
string str = s.ReadSizedString((long)length, TextUtils.GameTextEncoding.UTF8);
s.Position = p;
return(str);
}
else
{
// format is 8 bytes length, then [number read] bytes string
uint restBytes = maxBytes - 8;
if (length > restBytes)
{
// can't be a valid string
s.DiscardBytes(restBytes);
return(null);
}
string str = s.ReadSizedString((long)length, TextUtils.GameTextEncoding.UTF8);
s.DiscardBytes(restBytes - length);
return(str);
}
}
public static DeflateSharpCompressionInfo?Deserialize(DuplicatableStream stream, long maxBytes, EndianUtils.Endianness endian)
{
// note: identifier has already been read
if (maxBytes < 8)
{
stream.DiscardBytes(maxBytes);
return(null);
}
ulong uncompressedFilesize = stream.ReadUInt64(endian);
stream.DiscardBytes(maxBytes - 8);
return(new DeflateSharpCompressionInfo(uncompressedFilesize));
}
public static HyoutaArchiveBpsPatchInfo?Deserialize(DuplicatableStream stream, long maxBytes, EndianUtils.Endianness endian, ulong currentFileIndex, HyoutaArchiveChunk referencedChunk)
{
if (maxBytes < 16)
{
stream.DiscardBytes(maxBytes);
return(null);
}
else
{
ulong fileIndexToPatch = stream.ReadUInt64(endian);
ulong targetFilesize = stream.ReadUInt64(endian);
stream.DiscardBytes(maxBytes - 16);
if (fileIndexToPatch == currentFileIndex)
{
// this is how you set a file to be unpatched in an archive that has patches
return(null);
}
else
{
return(new HyoutaArchiveBpsPatchInfo(fileIndexToPatch, targetFilesize, referencedChunk));
}
}
}
public HyoutaArchiveChunk(DuplicatableStream duplicatableStream, out ulong chunkLength)
{
using (DuplicatableStream data = duplicatableStream.Duplicate()) {
data.Position = 0;
// header
ulong extraMagic = data.ReadUInt64(EndianUtils.Endianness.LittleEndian);
ulong magic = extraMagic & 0x00fffffffffffffful;
if (magic != 0x6b6e7568636168)
{
throw new Exception("wrong magic");
}
byte extra = (byte)((extraMagic >> 56) & 0xffu);
byte packedAlignment = (byte)(extra & 0x1fu);
long unpackedAlignment = 1L << packedAlignment;
bool hasMetadata = (extra & 0x20) != 0;
bool isCompressed = (extra & 0x40) != 0;
bool isBigEndian = (extra & 0x80) != 0;
EndianUtils.Endianness e = isBigEndian ? EndianUtils.Endianness.BigEndian : EndianUtils.Endianness.LittleEndian;
ulong endOfFileOffset = data.ReadUInt64(e) << packedAlignment;
ulong tableOfContentsOffset = data.ReadUInt64(e) << packedAlignment;
ulong filecount = data.ReadUInt64(e);
chunkLength = endOfFileOffset;
if (hasMetadata)
{
// just skip past this for now
ulong metadataLength = data.ReadUInt64(e);
data.DiscardBytes(metadataLength);
}
DuplicatableStream dataBlockStream;
if (isCompressed)
{
ushort compressionInfoLengthRaw = data.ReadUInt16(e);
uint compressionInfoLength = compressionInfoLengthRaw & 0xfffcu;
int compressionInfoAlignmentPacked = (compressionInfoLengthRaw & 0x3) + 1;
data.ReadAlign(1u << compressionInfoAlignmentPacked);
Compression.IHyoutaArchiveCompressionInfo?compressionInfo = HyoutaArchiveCompression.Deserialize(data, compressionInfoLength == 0 ? 0x10000u : compressionInfoLength, e);
if (compressionInfo == null)
{
throw new Exception("File is indicated to be compressed, but no decompressor found.");
}
dataBlockStream = compressionInfo.Decompress(data);
}
else
{
data.ReadAlign(unpackedAlignment);
dataBlockStream = new PartialStream(data, data.Position, (long)(endOfFileOffset - (ulong)data.Position));
}
try {
data.Dispose();
dataBlockStream.Position = (long)tableOfContentsOffset;
uint offsetToFirstFileInfo = ReadContentLength(dataBlockStream, e);
// decode content bitfield(s)
long numberOfUnknownBits = 0;
ushort contentBitfield1 = dataBlockStream.ReadUInt16(e);
bool hasDummyContent = (contentBitfield1 & 0x0001u) != 0;
bool hasFilename = (contentBitfield1 & 0x0002u) != 0;
bool hasCompression = (contentBitfield1 & 0x0004u) != 0;
bool hasBpsPatch = (contentBitfield1 & 0x0008u) != 0;
bool hasCrc32 = (contentBitfield1 & 0x0010u) != 0;
bool hasMd5 = (contentBitfield1 & 0x0020u) != 0;
bool hasSha1 = (contentBitfield1 & 0x0040u) != 0;
numberOfUnknownBits += (contentBitfield1 & 0x0080u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0100u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0200u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0400u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0800u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x1000u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x2000u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x4000u) != 0 ? 1 : 0;
ushort currentBitfield = contentBitfield1;
while ((currentBitfield & 0x8000u) != 0)
{
// more bitfields, though we don't understand them since only the first handful of bits are defined at the moment, so just count and skip them
currentBitfield = dataBlockStream.ReadUInt16(e);
numberOfUnknownBits += (currentBitfield & 0x0001u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0002u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0004u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0008u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0010u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0020u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0040u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0080u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0100u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0200u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0400u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0800u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x1000u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x2000u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x4000u) != 0 ? 1 : 0;
}
uint dummyContentLength = hasDummyContent ? ReadContentLength(dataBlockStream, e) : 0;
uint filenameLength = hasFilename ? ReadContentLength(dataBlockStream, e) : 0;
uint compressionLength = hasCompression ? ReadContentLength(dataBlockStream, e) : 0;
uint bpspatchLength = hasBpsPatch ? ReadContentLength(dataBlockStream, e) : 0;
uint crc32Length = hasCrc32 ? ReadContentLength(dataBlockStream, e) : 0;
uint md5Length = hasMd5 ? ReadContentLength(dataBlockStream, e) : 0;
uint sha1Length = hasSha1 ? ReadContentLength(dataBlockStream, e) : 0;
long unknownContentLength = 0;
for (long i = 0; i < numberOfUnknownBits; ++i)
{
unknownContentLength += ReadContentLength(dataBlockStream, e);
}
dataBlockStream.Position = (long)(tableOfContentsOffset + offsetToFirstFileInfo);
List <HyoutaArchiveFileInfo> files = new List <HyoutaArchiveFileInfo>((int)filecount);
for (ulong i = 0; i < filecount; ++i)
{
ulong offset = dataBlockStream.ReadUInt64(e) << packedAlignment;
ulong filesize = dataBlockStream.ReadUInt64(e);
HyoutaArchiveFileInfo fi = new HyoutaArchiveFileInfo();
if (hasDummyContent)
{
fi.DummyContent = dataBlockStream.ReadBytes(dummyContentLength);
}
if (hasFilename)
{
fi.Filename = ReadString(dataBlockStream, filenameLength, e);
}
if (hasCompression)
{
fi.CompressionInfo = HyoutaArchiveCompression.Deserialize(dataBlockStream, compressionLength, e);
fi.StreamIsCompressed = true;
}
if (hasBpsPatch)
{
fi.BpsPatchInfo = HyoutaArchiveBpsPatchInfo.Deserialize(dataBlockStream, bpspatchLength, e, i, this);
fi.StreamIsBpsPatch = fi.BpsPatchInfo != null;
}
if (hasCrc32)
{
if (crc32Length >= 4)
{
fi.crc32 = new CRC32(dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian));
dataBlockStream.DiscardBytes(crc32Length - 4);
}
else
{
dataBlockStream.DiscardBytes(crc32Length);
}
}
if (hasMd5)
{
if (md5Length >= 16)
{
ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
fi.md5 = new MD5(a, b);
dataBlockStream.DiscardBytes(md5Length - 16);
}
else
{
dataBlockStream.DiscardBytes(md5Length);
}
}
if (hasSha1)
{
if (sha1Length >= 20)
{
ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
uint c = dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian);
fi.sha1 = new SHA1(a, b, c);
dataBlockStream.DiscardBytes(sha1Length - 20);
}
else
{
dataBlockStream.DiscardBytes(sha1Length);
}
}
dataBlockStream.DiscardBytes(unknownContentLength);
fi.Data = new PartialStream(dataBlockStream, (long)offset, (long)filesize);
files.Add(fi);
}
Files = files;
} finally {
dataBlockStream.Dispose();
}
}
}
public UtfBuilder(DuplicatableStream stream, Endianness endian = Endianness.BigEndian)
{
long utfHeaderOffset = stream.Position;
uint utfMagic = stream.ReadUInt32(Endianness.LittleEndian);
if (utfMagic != 0x46545540)
{
throw new Exception("wrong UTF magic");
}
uint size = stream.ReadUInt32(endian); // size of the whole UTF chunk minus the magic and this value itself
// offsets are relative to here
long offset = stream.Position;
uint rowsLocation = stream.ReadUInt32(endian);
uint stringTableLocation = stream.ReadUInt32(endian);
uint dataTableLocation = stream.ReadUInt32(endian);
uint tableNameLocationInStringTable = stream.ReadUInt32(endian);
ushort colcount = stream.ReadUInt16(endian);
ushort rowwidth = stream.ReadUInt16(endian);
uint rowcount = stream.ReadUInt32(endian);
string tableName = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + tableNameLocationInStringTable);
// utf_tab calls this the 'schema'
List <ColumnData> columns = new List <ColumnData>(colcount);
for (int i = 0; i < colcount; ++i)
{
ColumnData col = new ColumnData();
col.Type = stream.ReadUInt8();
uint stroffset = stream.ReadUInt32(endian);
col.Name = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + stroffset);
if ((col.Type & utf_tab_sharp.UtfTab.COLUMN_STORAGE_MASK) == utf_tab_sharp.UtfTab.COLUMN_STORAGE_CONSTANT)
{
switch (col.Type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK)
{
case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING:
uint datastroffset = stream.ReadUInt32(endian);
col.Data = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + datastroffset);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE:
col.Data = stream.ReadUInt64(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA:
uint dataOffset = stream.ReadUInt32(endian);
uint dataSize = stream.ReadUInt32(endian);
col.Data = stream.ReadBytesFromLocationAndReset(offset + dataTableLocation + dataOffset, dataSize);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT:
col.Data = stream.ReadUInt32(endian).UIntToFloat();
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE:
col.Data = stream.ReadUInt32(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE:
col.Data = stream.ReadUInt16(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE:
col.Data = stream.ReadUInt8();
break;
default:
throw new Exception("unknown type for constant");
}
}
columns.Add(col);
}
List <RowData> rows = new List <RowData>((int)rowcount);
for (long i = 0; i < rowcount; ++i)
{
stream.Position = offset + rowsLocation + (i * rowwidth);
RowData row = new RowData();
row.Cells = new List <CellData>();
for (int j = 0; j < colcount; ++j)
{
CellData cell = new CellData();
byte type = columns[j].Type;
switch ((type & utf_tab_sharp.UtfTab.COLUMN_STORAGE_MASK))
{
case utf_tab_sharp.UtfTab.COLUMN_STORAGE_PERROW:
switch (type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK)
{
case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING:
uint datastroffset = stream.ReadUInt32(endian);
cell.Data = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + datastroffset);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE:
cell.Data = stream.ReadUInt64(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA:
uint dataOffset = stream.ReadUInt32(endian);
uint dataSize = stream.ReadUInt32(endian);
cell.Data = stream.ReadBytesFromLocationAndReset(offset + dataTableLocation + dataOffset, dataSize);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT:
cell.Data = stream.ReadUInt32(endian).UIntToFloat();
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE:
cell.Data = stream.ReadUInt32(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE:
cell.Data = stream.ReadUInt16(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE:
cell.Data = stream.ReadUInt8();
break;
default:
throw new Exception("unknown type for value");
}
break;
case utf_tab_sharp.UtfTab.COLUMN_STORAGE_CONSTANT:
cell.Data = columns[j].Data;
break;
case utf_tab_sharp.UtfTab.COLUMN_STORAGE_ZERO:
switch (type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK)
{
case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING:
cell.Data = "";
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE:
cell.Data = (ulong)0;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA:
cell.Data = new byte[0];
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT:
cell.Data = 0.0f;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE:
cell.Data = (uint)0;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE:
cell.Data = (ushort)0;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE:
cell.Data = (byte)0;
break;
default:
throw new Exception("unknown type for value");
}
break;
default:
throw new Exception("unknown storage for value");
}
row.Cells.Add(cell);
}
rows.Add(row);
}
Name = tableName;
Columns = columns;
Rows = rows;
stream.Position = utfHeaderOffset + ((long)8) + ((long)size);
return;
}