public static Compression.IHyoutaArchiveCompressionInfo?Deserialize(DuplicatableStream stream, long maxBytes, EndianUtils.Endianness endian) { if (maxBytes < 8) { stream.DiscardBytes(maxBytes); return(null); } ulong identifier = stream.ReadUInt64(EndianUtils.Endianness.BigEndian); switch (identifier) { case 0: // archive has compression, but this file is not compressed stream.DiscardBytes(maxBytes - 8); return(null); case DeflateSharpCompressionInfo.Identifier: return(DeflateSharpCompressionInfo.Deserialize(stream, maxBytes - 8, endian)); default: Console.WriteLine("Unknown compression type: " + identifier.ToString("x16")); stream.DiscardBytes(maxBytes - 8); return(null); } }
public EndianUtils.Endianness Endian; // not actually a field in the header, at least not here -- might be the first byte? public NubHeader(DuplicatableStream stream, EndianUtils.Endianness?endian) { Magic = stream.ReadUInt64(EndianUtils.Endianness.LittleEndian); EndianUtils.Endianness suspectedEndian; if (Magic == 0x10200) { suspectedEndian = EndianUtils.Endianness.BigEndian; } else if (Magic == 0x10201) { suspectedEndian = EndianUtils.Endianness.LittleEndian; } else { throw new Exception("unexpected magic in NUB"); } EndianUtils.Endianness e = endian ?? suspectedEndian; Fileid = stream.ReadUInt32(e); // or something like that? seems unique per archive in each game EntryCount = stream.ReadUInt32(e); StartOfFiles = stream.ReadUInt32(e); FilesSize = stream.ReadUInt32(e); StartOfEntries = stream.ReadUInt32(e); StartOfHeaders = stream.ReadUInt32(e); Endian = e; }
private static string?ReadString(DuplicatableStream s, uint maxBytes, EndianUtils.Endianness e) { if (maxBytes < 8) { // can't be a valid string s.DiscardBytes(maxBytes); return(null); } ulong rawlength = s.ReadUInt64(e); ulong length = (rawlength & 0x7ffffffffffffffful); bool hasOffset = (rawlength & 0x8000000000000000ul) > 0; if (hasOffset) { // format is 8 bytes length, then 8 bytes position of string in data if (maxBytes < 16) { // can't be valid s.DiscardBytes(maxBytes - 8); return(null); } ulong offset = s.ReadUInt64(e); long p = s.Position + (maxBytes - 16); s.Position = (long)offset; string str = s.ReadSizedString((long)length, TextUtils.GameTextEncoding.UTF8); s.Position = p; return(str); } else { // format is 8 bytes length, then [number read] bytes string uint restBytes = maxBytes - 8; if (length > restBytes) { // can't be a valid string s.DiscardBytes(restBytes); return(null); } string str = s.ReadSizedString((long)length, TextUtils.GameTextEncoding.UTF8); s.DiscardBytes(restBytes - length); return(str); } }
public static DeflateSharpCompressionInfo?Deserialize(DuplicatableStream stream, long maxBytes, EndianUtils.Endianness endian) { // note: identifier has already been read if (maxBytes < 8) { stream.DiscardBytes(maxBytes); return(null); } ulong uncompressedFilesize = stream.ReadUInt64(endian); stream.DiscardBytes(maxBytes - 8); return(new DeflateSharpCompressionInfo(uncompressedFilesize)); }
public static HyoutaArchiveBpsPatchInfo?Deserialize(DuplicatableStream stream, long maxBytes, EndianUtils.Endianness endian, ulong currentFileIndex, HyoutaArchiveChunk referencedChunk) { if (maxBytes < 16) { stream.DiscardBytes(maxBytes); return(null); } else { ulong fileIndexToPatch = stream.ReadUInt64(endian); ulong targetFilesize = stream.ReadUInt64(endian); stream.DiscardBytes(maxBytes - 16); if (fileIndexToPatch == currentFileIndex) { // this is how you set a file to be unpatched in an archive that has patches return(null); } else { return(new HyoutaArchiveBpsPatchInfo(fileIndexToPatch, targetFilesize, referencedChunk)); } } }
public HyoutaArchiveChunk(DuplicatableStream duplicatableStream, out ulong chunkLength) { using (DuplicatableStream data = duplicatableStream.Duplicate()) { data.Position = 0; // header ulong extraMagic = data.ReadUInt64(EndianUtils.Endianness.LittleEndian); ulong magic = extraMagic & 0x00fffffffffffffful; if (magic != 0x6b6e7568636168) { throw new Exception("wrong magic"); } byte extra = (byte)((extraMagic >> 56) & 0xffu); byte packedAlignment = (byte)(extra & 0x1fu); long unpackedAlignment = 1L << packedAlignment; bool hasMetadata = (extra & 0x20) != 0; bool isCompressed = (extra & 0x40) != 0; bool isBigEndian = (extra & 0x80) != 0; EndianUtils.Endianness e = isBigEndian ? EndianUtils.Endianness.BigEndian : EndianUtils.Endianness.LittleEndian; ulong endOfFileOffset = data.ReadUInt64(e) << packedAlignment; ulong tableOfContentsOffset = data.ReadUInt64(e) << packedAlignment; ulong filecount = data.ReadUInt64(e); chunkLength = endOfFileOffset; if (hasMetadata) { // just skip past this for now ulong metadataLength = data.ReadUInt64(e); data.DiscardBytes(metadataLength); } DuplicatableStream dataBlockStream; if (isCompressed) { ushort compressionInfoLengthRaw = data.ReadUInt16(e); uint compressionInfoLength = compressionInfoLengthRaw & 0xfffcu; int compressionInfoAlignmentPacked = (compressionInfoLengthRaw & 0x3) + 1; data.ReadAlign(1u << compressionInfoAlignmentPacked); Compression.IHyoutaArchiveCompressionInfo?compressionInfo = HyoutaArchiveCompression.Deserialize(data, compressionInfoLength == 0 ? 0x10000u : compressionInfoLength, e); if (compressionInfo == null) { throw new Exception("File is indicated to be compressed, but no decompressor found."); } dataBlockStream = compressionInfo.Decompress(data); } else { data.ReadAlign(unpackedAlignment); dataBlockStream = new PartialStream(data, data.Position, (long)(endOfFileOffset - (ulong)data.Position)); } try { data.Dispose(); dataBlockStream.Position = (long)tableOfContentsOffset; uint offsetToFirstFileInfo = ReadContentLength(dataBlockStream, e); // decode content bitfield(s) long numberOfUnknownBits = 0; ushort contentBitfield1 = dataBlockStream.ReadUInt16(e); bool hasDummyContent = (contentBitfield1 & 0x0001u) != 0; bool hasFilename = (contentBitfield1 & 0x0002u) != 0; bool hasCompression = (contentBitfield1 & 0x0004u) != 0; bool hasBpsPatch = (contentBitfield1 & 0x0008u) != 0; bool hasCrc32 = (contentBitfield1 & 0x0010u) != 0; bool hasMd5 = (contentBitfield1 & 0x0020u) != 0; bool hasSha1 = (contentBitfield1 & 0x0040u) != 0; numberOfUnknownBits += (contentBitfield1 & 0x0080u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x0100u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x0200u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x0400u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x0800u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x1000u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x2000u) != 0 ? 1 : 0; numberOfUnknownBits += (contentBitfield1 & 0x4000u) != 0 ? 1 : 0; ushort currentBitfield = contentBitfield1; while ((currentBitfield & 0x8000u) != 0) { // more bitfields, though we don't understand them since only the first handful of bits are defined at the moment, so just count and skip them currentBitfield = dataBlockStream.ReadUInt16(e); numberOfUnknownBits += (currentBitfield & 0x0001u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0002u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0004u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0008u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0010u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0020u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0040u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0080u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0100u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0200u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0400u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x0800u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x1000u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x2000u) != 0 ? 1 : 0; numberOfUnknownBits += (currentBitfield & 0x4000u) != 0 ? 1 : 0; } uint dummyContentLength = hasDummyContent ? ReadContentLength(dataBlockStream, e) : 0; uint filenameLength = hasFilename ? ReadContentLength(dataBlockStream, e) : 0; uint compressionLength = hasCompression ? ReadContentLength(dataBlockStream, e) : 0; uint bpspatchLength = hasBpsPatch ? ReadContentLength(dataBlockStream, e) : 0; uint crc32Length = hasCrc32 ? ReadContentLength(dataBlockStream, e) : 0; uint md5Length = hasMd5 ? ReadContentLength(dataBlockStream, e) : 0; uint sha1Length = hasSha1 ? ReadContentLength(dataBlockStream, e) : 0; long unknownContentLength = 0; for (long i = 0; i < numberOfUnknownBits; ++i) { unknownContentLength += ReadContentLength(dataBlockStream, e); } dataBlockStream.Position = (long)(tableOfContentsOffset + offsetToFirstFileInfo); List <HyoutaArchiveFileInfo> files = new List <HyoutaArchiveFileInfo>((int)filecount); for (ulong i = 0; i < filecount; ++i) { ulong offset = dataBlockStream.ReadUInt64(e) << packedAlignment; ulong filesize = dataBlockStream.ReadUInt64(e); HyoutaArchiveFileInfo fi = new HyoutaArchiveFileInfo(); if (hasDummyContent) { fi.DummyContent = dataBlockStream.ReadBytes(dummyContentLength); } if (hasFilename) { fi.Filename = ReadString(dataBlockStream, filenameLength, e); } if (hasCompression) { fi.CompressionInfo = HyoutaArchiveCompression.Deserialize(dataBlockStream, compressionLength, e); fi.StreamIsCompressed = true; } if (hasBpsPatch) { fi.BpsPatchInfo = HyoutaArchiveBpsPatchInfo.Deserialize(dataBlockStream, bpspatchLength, e, i, this); fi.StreamIsBpsPatch = fi.BpsPatchInfo != null; } if (hasCrc32) { if (crc32Length >= 4) { fi.crc32 = new CRC32(dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian)); dataBlockStream.DiscardBytes(crc32Length - 4); } else { dataBlockStream.DiscardBytes(crc32Length); } } if (hasMd5) { if (md5Length >= 16) { ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian); ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian); fi.md5 = new MD5(a, b); dataBlockStream.DiscardBytes(md5Length - 16); } else { dataBlockStream.DiscardBytes(md5Length); } } if (hasSha1) { if (sha1Length >= 20) { ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian); ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian); uint c = dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian); fi.sha1 = new SHA1(a, b, c); dataBlockStream.DiscardBytes(sha1Length - 20); } else { dataBlockStream.DiscardBytes(sha1Length); } } dataBlockStream.DiscardBytes(unknownContentLength); fi.Data = new PartialStream(dataBlockStream, (long)offset, (long)filesize); files.Add(fi); } Files = files; } finally { dataBlockStream.Dispose(); } } }
public UtfBuilder(DuplicatableStream stream, Endianness endian = Endianness.BigEndian) { long utfHeaderOffset = stream.Position; uint utfMagic = stream.ReadUInt32(Endianness.LittleEndian); if (utfMagic != 0x46545540) { throw new Exception("wrong UTF magic"); } uint size = stream.ReadUInt32(endian); // size of the whole UTF chunk minus the magic and this value itself // offsets are relative to here long offset = stream.Position; uint rowsLocation = stream.ReadUInt32(endian); uint stringTableLocation = stream.ReadUInt32(endian); uint dataTableLocation = stream.ReadUInt32(endian); uint tableNameLocationInStringTable = stream.ReadUInt32(endian); ushort colcount = stream.ReadUInt16(endian); ushort rowwidth = stream.ReadUInt16(endian); uint rowcount = stream.ReadUInt32(endian); string tableName = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + tableNameLocationInStringTable); // utf_tab calls this the 'schema' List <ColumnData> columns = new List <ColumnData>(colcount); for (int i = 0; i < colcount; ++i) { ColumnData col = new ColumnData(); col.Type = stream.ReadUInt8(); uint stroffset = stream.ReadUInt32(endian); col.Name = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + stroffset); if ((col.Type & utf_tab_sharp.UtfTab.COLUMN_STORAGE_MASK) == utf_tab_sharp.UtfTab.COLUMN_STORAGE_CONSTANT) { switch (col.Type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK) { case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING: uint datastroffset = stream.ReadUInt32(endian); col.Data = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + datastroffset); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE: col.Data = stream.ReadUInt64(endian); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA: uint dataOffset = stream.ReadUInt32(endian); uint dataSize = stream.ReadUInt32(endian); col.Data = stream.ReadBytesFromLocationAndReset(offset + dataTableLocation + dataOffset, dataSize); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT: col.Data = stream.ReadUInt32(endian).UIntToFloat(); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE: col.Data = stream.ReadUInt32(endian); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE: col.Data = stream.ReadUInt16(endian); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE: col.Data = stream.ReadUInt8(); break; default: throw new Exception("unknown type for constant"); } } columns.Add(col); } List <RowData> rows = new List <RowData>((int)rowcount); for (long i = 0; i < rowcount; ++i) { stream.Position = offset + rowsLocation + (i * rowwidth); RowData row = new RowData(); row.Cells = new List <CellData>(); for (int j = 0; j < colcount; ++j) { CellData cell = new CellData(); byte type = columns[j].Type; switch ((type & utf_tab_sharp.UtfTab.COLUMN_STORAGE_MASK)) { case utf_tab_sharp.UtfTab.COLUMN_STORAGE_PERROW: switch (type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK) { case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING: uint datastroffset = stream.ReadUInt32(endian); cell.Data = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + datastroffset); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE: cell.Data = stream.ReadUInt64(endian); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA: uint dataOffset = stream.ReadUInt32(endian); uint dataSize = stream.ReadUInt32(endian); cell.Data = stream.ReadBytesFromLocationAndReset(offset + dataTableLocation + dataOffset, dataSize); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT: cell.Data = stream.ReadUInt32(endian).UIntToFloat(); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE: cell.Data = stream.ReadUInt32(endian); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE: cell.Data = stream.ReadUInt16(endian); break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE: cell.Data = stream.ReadUInt8(); break; default: throw new Exception("unknown type for value"); } break; case utf_tab_sharp.UtfTab.COLUMN_STORAGE_CONSTANT: cell.Data = columns[j].Data; break; case utf_tab_sharp.UtfTab.COLUMN_STORAGE_ZERO: switch (type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK) { case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING: cell.Data = ""; break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE: cell.Data = (ulong)0; break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA: cell.Data = new byte[0]; break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT: cell.Data = 0.0f; break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE: cell.Data = (uint)0; break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE: cell.Data = (ushort)0; break; case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2: case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE: cell.Data = (byte)0; break; default: throw new Exception("unknown type for value"); } break; default: throw new Exception("unknown storage for value"); } row.Cells.Add(cell); } rows.Add(row); } Name = tableName; Columns = columns; Rows = rows; stream.Position = utfHeaderOffset + ((long)8) + ((long)size); return; }