private bool LoadFile(DuplicatableStream headerStream, DuplicatableStream contentStream)
{
DuplicatableStream infile = headerStream.Duplicate();
contentFile = contentStream.Duplicate();
infile.Seek(0x00, SeekOrigin.Begin);
string magic = infile.ReadAscii(4);
if (magic != "FPS4")
{
Console.WriteLine("Not an FPS4 file!");
return(false);
}
Endian = Util.Endianness.BigEndian;
FileCount = infile.ReadUInt32().FromEndian(Endian);
HeaderSize = infile.ReadUInt32().FromEndian(Endian);
// if header seems huge then we probably have assumed the wrong endianness
if (HeaderSize > 0xFFFF)
{
Endian = Util.Endianness.LittleEndian;
FileCount = FileCount.ToEndian(Util.Endianness.BigEndian).FromEndian(Endian);
HeaderSize = HeaderSize.ToEndian(Util.Endianness.BigEndian).FromEndian(Endian);
}
FirstFileStart = infile.ReadUInt32().FromEndian(Endian);
EntrySize = infile.ReadUInt16().FromEndian(Endian);
ContentBitmask = new ContentInfo(infile.ReadUInt16().FromEndian(Endian));
Unknown2 = infile.ReadUInt32().FromEndian(Endian);
ArchiveNameLocation = infile.ReadUInt32().FromEndian(Endian);
infile.Position = ArchiveNameLocation;
if (ArchiveNameLocation > 0)
{
ArchiveName = infile.ReadShiftJisNullterm();
}
Alignment = FirstFileStart;
Console.WriteLine("Content Bitmask: 0x" + ContentBitmask.Value.ToString("X4"));
if (ContentBitmask.HasUnknownDataTypes)
{
Console.WriteLine("WARNING: Bitmask identifies unknown data types, data interpretation will probably be incorrect.");
}
Files = new List <FileInfo>((int)FileCount);
for (uint i = 0; i < FileCount; ++i)
{
infile.Position = HeaderSize + (i * EntrySize);
Files.Add(new FileInfo(infile, i, ContentBitmask, Endian, Util.GameTextEncoding.ASCII));
}
FileLocationMultiplier = CalculateFileLocationMultiplier();
ShouldGuessFilesizeFromNextFile = !ContentBitmask.ContainsFileSizes && !ContentBitmask.ContainsSectorSizes && CalculateIsLinear();
infile.Dispose();
return(true);
}
public HyoutaArchiveChunk(DuplicatableStream duplicatableStream, out ulong chunkLength)
{
using (DuplicatableStream data = duplicatableStream.Duplicate()) {
data.Position = 0;
// header
ulong extraMagic = data.ReadUInt64(EndianUtils.Endianness.LittleEndian);
ulong magic = extraMagic & 0x00fffffffffffffful;
if (magic != 0x6b6e7568636168)
{
throw new Exception("wrong magic");
}
byte extra = (byte)((extraMagic >> 56) & 0xffu);
byte packedAlignment = (byte)(extra & 0x1fu);
long unpackedAlignment = 1L << packedAlignment;
bool hasMetadata = (extra & 0x20) != 0;
bool isCompressed = (extra & 0x40) != 0;
bool isBigEndian = (extra & 0x80) != 0;
EndianUtils.Endianness e = isBigEndian ? EndianUtils.Endianness.BigEndian : EndianUtils.Endianness.LittleEndian;
ulong endOfFileOffset = data.ReadUInt64(e) << packedAlignment;
ulong tableOfContentsOffset = data.ReadUInt64(e) << packedAlignment;
ulong filecount = data.ReadUInt64(e);
chunkLength = endOfFileOffset;
if (hasMetadata)
{
// just skip past this for now
ulong metadataLength = data.ReadUInt64(e);
data.DiscardBytes(metadataLength);
}
DuplicatableStream dataBlockStream;
if (isCompressed)
{
ushort compressionInfoLengthRaw = data.ReadUInt16(e);
uint compressionInfoLength = compressionInfoLengthRaw & 0xfffcu;
int compressionInfoAlignmentPacked = (compressionInfoLengthRaw & 0x3) + 1;
data.ReadAlign(1u << compressionInfoAlignmentPacked);
Compression.IHyoutaArchiveCompressionInfo?compressionInfo = HyoutaArchiveCompression.Deserialize(data, compressionInfoLength == 0 ? 0x10000u : compressionInfoLength, e);
if (compressionInfo == null)
{
throw new Exception("File is indicated to be compressed, but no decompressor found.");
}
dataBlockStream = compressionInfo.Decompress(data);
}
else
{
data.ReadAlign(unpackedAlignment);
dataBlockStream = new PartialStream(data, data.Position, (long)(endOfFileOffset - (ulong)data.Position));
}
try {
data.Dispose();
dataBlockStream.Position = (long)tableOfContentsOffset;
uint offsetToFirstFileInfo = ReadContentLength(dataBlockStream, e);
// decode content bitfield(s)
long numberOfUnknownBits = 0;
ushort contentBitfield1 = dataBlockStream.ReadUInt16(e);
bool hasDummyContent = (contentBitfield1 & 0x0001u) != 0;
bool hasFilename = (contentBitfield1 & 0x0002u) != 0;
bool hasCompression = (contentBitfield1 & 0x0004u) != 0;
bool hasBpsPatch = (contentBitfield1 & 0x0008u) != 0;
bool hasCrc32 = (contentBitfield1 & 0x0010u) != 0;
bool hasMd5 = (contentBitfield1 & 0x0020u) != 0;
bool hasSha1 = (contentBitfield1 & 0x0040u) != 0;
numberOfUnknownBits += (contentBitfield1 & 0x0080u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0100u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0200u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0400u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0800u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x1000u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x2000u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x4000u) != 0 ? 1 : 0;
ushort currentBitfield = contentBitfield1;
while ((currentBitfield & 0x8000u) != 0)
{
// more bitfields, though we don't understand them since only the first handful of bits are defined at the moment, so just count and skip them
currentBitfield = dataBlockStream.ReadUInt16(e);
numberOfUnknownBits += (currentBitfield & 0x0001u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0002u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0004u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0008u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0010u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0020u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0040u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0080u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0100u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0200u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0400u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0800u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x1000u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x2000u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x4000u) != 0 ? 1 : 0;
}
uint dummyContentLength = hasDummyContent ? ReadContentLength(dataBlockStream, e) : 0;
uint filenameLength = hasFilename ? ReadContentLength(dataBlockStream, e) : 0;
uint compressionLength = hasCompression ? ReadContentLength(dataBlockStream, e) : 0;
uint bpspatchLength = hasBpsPatch ? ReadContentLength(dataBlockStream, e) : 0;
uint crc32Length = hasCrc32 ? ReadContentLength(dataBlockStream, e) : 0;
uint md5Length = hasMd5 ? ReadContentLength(dataBlockStream, e) : 0;
uint sha1Length = hasSha1 ? ReadContentLength(dataBlockStream, e) : 0;
long unknownContentLength = 0;
for (long i = 0; i < numberOfUnknownBits; ++i)
{
unknownContentLength += ReadContentLength(dataBlockStream, e);
}
dataBlockStream.Position = (long)(tableOfContentsOffset + offsetToFirstFileInfo);
List <HyoutaArchiveFileInfo> files = new List <HyoutaArchiveFileInfo>((int)filecount);
for (ulong i = 0; i < filecount; ++i)
{
ulong offset = dataBlockStream.ReadUInt64(e) << packedAlignment;
ulong filesize = dataBlockStream.ReadUInt64(e);
HyoutaArchiveFileInfo fi = new HyoutaArchiveFileInfo();
if (hasDummyContent)
{
fi.DummyContent = dataBlockStream.ReadBytes(dummyContentLength);
}
if (hasFilename)
{
fi.Filename = ReadString(dataBlockStream, filenameLength, e);
}
if (hasCompression)
{
fi.CompressionInfo = HyoutaArchiveCompression.Deserialize(dataBlockStream, compressionLength, e);
fi.StreamIsCompressed = true;
}
if (hasBpsPatch)
{
fi.BpsPatchInfo = HyoutaArchiveBpsPatchInfo.Deserialize(dataBlockStream, bpspatchLength, e, i, this);
fi.StreamIsBpsPatch = fi.BpsPatchInfo != null;
}
if (hasCrc32)
{
if (crc32Length >= 4)
{
fi.crc32 = new CRC32(dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian));
dataBlockStream.DiscardBytes(crc32Length - 4);
}
else
{
dataBlockStream.DiscardBytes(crc32Length);
}
}
if (hasMd5)
{
if (md5Length >= 16)
{
ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
fi.md5 = new MD5(a, b);
dataBlockStream.DiscardBytes(md5Length - 16);
}
else
{
dataBlockStream.DiscardBytes(md5Length);
}
}
if (hasSha1)
{
if (sha1Length >= 20)
{
ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
uint c = dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian);
fi.sha1 = new SHA1(a, b, c);
dataBlockStream.DiscardBytes(sha1Length - 20);
}
else
{
dataBlockStream.DiscardBytes(sha1Length);
}
}
dataBlockStream.DiscardBytes(unknownContentLength);
fi.Data = new PartialStream(dataBlockStream, (long)offset, (long)filesize);
files.Add(fi);
}
Files = files;
} finally {
dataBlockStream.Dispose();
}
}
}
private static uint ReadContentLength(DuplicatableStream dataBlockStream, EndianUtils.Endianness e)
{
ushort l = dataBlockStream.ReadUInt16(e);
return(l == 0 ? 0x10000 : (uint)l);
}
public UtfBuilder(DuplicatableStream stream, Endianness endian = Endianness.BigEndian)
{
long utfHeaderOffset = stream.Position;
uint utfMagic = stream.ReadUInt32(Endianness.LittleEndian);
if (utfMagic != 0x46545540)
{
throw new Exception("wrong UTF magic");
}
uint size = stream.ReadUInt32(endian); // size of the whole UTF chunk minus the magic and this value itself
// offsets are relative to here
long offset = stream.Position;
uint rowsLocation = stream.ReadUInt32(endian);
uint stringTableLocation = stream.ReadUInt32(endian);
uint dataTableLocation = stream.ReadUInt32(endian);
uint tableNameLocationInStringTable = stream.ReadUInt32(endian);
ushort colcount = stream.ReadUInt16(endian);
ushort rowwidth = stream.ReadUInt16(endian);
uint rowcount = stream.ReadUInt32(endian);
string tableName = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + tableNameLocationInStringTable);
// utf_tab calls this the 'schema'
List <ColumnData> columns = new List <ColumnData>(colcount);
for (int i = 0; i < colcount; ++i)
{
ColumnData col = new ColumnData();
col.Type = stream.ReadUInt8();
uint stroffset = stream.ReadUInt32(endian);
col.Name = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + stroffset);
if ((col.Type & utf_tab_sharp.UtfTab.COLUMN_STORAGE_MASK) == utf_tab_sharp.UtfTab.COLUMN_STORAGE_CONSTANT)
{
switch (col.Type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK)
{
case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING:
uint datastroffset = stream.ReadUInt32(endian);
col.Data = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + datastroffset);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE:
col.Data = stream.ReadUInt64(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA:
uint dataOffset = stream.ReadUInt32(endian);
uint dataSize = stream.ReadUInt32(endian);
col.Data = stream.ReadBytesFromLocationAndReset(offset + dataTableLocation + dataOffset, dataSize);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT:
col.Data = stream.ReadUInt32(endian).UIntToFloat();
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE:
col.Data = stream.ReadUInt32(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE:
col.Data = stream.ReadUInt16(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE:
col.Data = stream.ReadUInt8();
break;
default:
throw new Exception("unknown type for constant");
}
}
columns.Add(col);
}
List <RowData> rows = new List <RowData>((int)rowcount);
for (long i = 0; i < rowcount; ++i)
{
stream.Position = offset + rowsLocation + (i * rowwidth);
RowData row = new RowData();
row.Cells = new List <CellData>();
for (int j = 0; j < colcount; ++j)
{
CellData cell = new CellData();
byte type = columns[j].Type;
switch ((type & utf_tab_sharp.UtfTab.COLUMN_STORAGE_MASK))
{
case utf_tab_sharp.UtfTab.COLUMN_STORAGE_PERROW:
switch (type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK)
{
case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING:
uint datastroffset = stream.ReadUInt32(endian);
cell.Data = stream.ReadUTF8NulltermFromLocationAndReset(offset + stringTableLocation + datastroffset);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE:
cell.Data = stream.ReadUInt64(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA:
uint dataOffset = stream.ReadUInt32(endian);
uint dataSize = stream.ReadUInt32(endian);
cell.Data = stream.ReadBytesFromLocationAndReset(offset + dataTableLocation + dataOffset, dataSize);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT:
cell.Data = stream.ReadUInt32(endian).UIntToFloat();
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE:
cell.Data = stream.ReadUInt32(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE:
cell.Data = stream.ReadUInt16(endian);
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE:
cell.Data = stream.ReadUInt8();
break;
default:
throw new Exception("unknown type for value");
}
break;
case utf_tab_sharp.UtfTab.COLUMN_STORAGE_CONSTANT:
cell.Data = columns[j].Data;
break;
case utf_tab_sharp.UtfTab.COLUMN_STORAGE_ZERO:
switch (type & utf_tab_sharp.UtfTab.COLUMN_TYPE_MASK)
{
case utf_tab_sharp.UtfTab.COLUMN_TYPE_STRING:
cell.Data = "";
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_8BYTE:
cell.Data = (ulong)0;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_DATA:
cell.Data = new byte[0];
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_FLOAT:
cell.Data = 0.0f;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_4BYTE:
cell.Data = (uint)0;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_2BYTE:
cell.Data = (ushort)0;
break;
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE2:
case utf_tab_sharp.UtfTab.COLUMN_TYPE_1BYTE:
cell.Data = (byte)0;
break;
default:
throw new Exception("unknown type for value");
}
break;
default:
throw new Exception("unknown storage for value");
}
row.Cells.Add(cell);
}
rows.Add(row);
}
Name = tableName;
Columns = columns;
Rows = rows;
stream.Position = utfHeaderOffset + ((long)8) + ((long)size);
return;
}