public static DuplicatableStream ApplyPatch(HyoutaArchiveBpsPatchInfo patchInfo, Stream data)
{
var refchunk = patchInfo.ReferencedChunk;
if (refchunk == null)
{
// implies that the file points at itself and is unpatched
return(data.CopyToByteArrayStream());
}
HyoutaArchiveFileInfo sourceFile = refchunk.GetFile((long)patchInfo.FileIndexToPatch);
using (DuplicatableStream sourceStream = sourceFile.DataStream.Duplicate()) {
sourceStream.Position = 0;
data.Position = 0;
using (MemoryStream ms = new MemoryStream((int)patchInfo.TargetFilesize)) {
Bps.BpsPatcher.ApplyPatchToStream(sourceStream, data, ms);
return(ms.CopyToByteArrayStreamAndDispose());
}
}
}
private static ulong PackDataBlock(Stream target, List <HyoutaArchiveFileInfo> files, byte packedAlignment, EndianUtils.Endianness endian)
{
byte smallPackedAlignment = ToSmallPackedAlignment(packedAlignment);
long startPosition = target.Position;
target.WriteUInt16(0); // offsetToFirstFileInfo, fill in later
bool hasDummyContent = files.Any(x => x.DummyContent != null);
uint dummyContentLength = hasDummyContent ? ((uint)files.Max(x => x.DummyContent?.Length ?? 0)).Align(1 << smallPackedAlignment) : 0;
bool hasFilename = files.Any(x => x.Filename != null);
uint filenameLength = 0;
//bool embedFilenamesInFileInfo = false;
List <byte[]?>?encodedFilenames = null;
if (hasFilename)
{
// figure out whether we want the strings to embed into the fileinfo directly
// or whether to use an offset and write the string data at the end of the fileinfo
// note that if a string is <= 8 bytes we can always embed it as we'd need 8 bytes for the offset anyway
// so...
encodedFilenames = new List <byte[]?>(files.Count);
long longestBytecount = 0;
long totalBytecount = 0;
long filenameCountOver8Bytes = 0;
for (int i = 0; i < files.Count; ++i)
{
var currentFilename = files[i].Filename;
if (currentFilename == null)
{
encodedFilenames.Add(null);
}
else
{
byte[] stringbytes = EncodeString(currentFilename);
encodedFilenames.Add(stringbytes);
if (stringbytes.LongLength > 8)
{
longestBytecount = Math.Max(longestBytecount, stringbytes.LongLength);
totalBytecount += stringbytes.LongLength;
++filenameCountOver8Bytes;
}
}
}
// alright so we have, in practice, two options here
// - make filenameLength == 16, store strings that are longer than that offsetted
long nonEmbedSize = files.Count * 16 + totalBytecount.Align(1 << smallPackedAlignment);
// - make filenameLength long enough so all strings can be embedded
long embedSize = files.Count * (8 + longestBytecount).Align(1 << smallPackedAlignment);
// pick whatever results in a smaller file; on a tie embed
if (nonEmbedSize < embedSize)
{
//embedFilenamesInFileInfo = false;
filenameLength = 16;
}
else
{
//embedFilenamesInFileInfo = true;
filenameLength = (uint)(8 + longestBytecount).Align(1 << smallPackedAlignment);
}
}
bool hasCompression = files.Any(x => x.CompressionInfo != null);
uint compressionInfoLength = hasCompression ? files.Max(x => x.CompressionInfo?.MaximumCompressionInfoLength() ?? 0).Align(1 << smallPackedAlignment) : 0;
bool hasBpsPatch = files.Any(x => x.BpsPatchInfo != null);
uint bpsPatchInfoLength = hasBpsPatch ? 16u.Align(1 << smallPackedAlignment) : 0;
bool hasCrc32 = files.Any(x => x.crc32 != null);
uint crc32ContentLength = hasCrc32 ? 4u.Align(1 << smallPackedAlignment) : 0u;
bool hasMd5 = files.Any(x => x.md5 != null);
uint md5ContentLength = hasMd5 ? 16u.Align(1 << smallPackedAlignment) : 0u;
bool hasSha1 = files.Any(x => x.sha1 != null);
uint sha1ContentLength = hasSha1 ? 20u.Align(1 << smallPackedAlignment) : 0u;
ushort contentBitfield1 = 0;
contentBitfield1 |= (ushort)(hasDummyContent ? 0x0001u : 0);
contentBitfield1 |= (ushort)(hasFilename ? 0x0002u : 0);
contentBitfield1 |= (ushort)(hasCompression ? 0x0004u : 0);
contentBitfield1 |= (ushort)(hasBpsPatch ? 0x0008u : 0);
contentBitfield1 |= (ushort)(hasCrc32 ? 0x0010u : 0);
contentBitfield1 |= (ushort)(hasMd5 ? 0x0020u : 0);
contentBitfield1 |= (ushort)(hasSha1 ? 0x0040u : 0);
target.WriteUInt16(contentBitfield1, endian);
if (hasDummyContent)
{
WriteContentLength(dummyContentLength, target, endian);
}
if (hasFilename)
{
WriteContentLength(filenameLength, target, endian);
}
if (hasCompression)
{
WriteContentLength(compressionInfoLength, target, endian);
}
if (hasBpsPatch)
{
WriteContentLength(bpsPatchInfoLength, target, endian);
}
if (hasCrc32)
{
WriteContentLength(crc32ContentLength, target, endian);
}
if (hasMd5)
{
WriteContentLength(md5ContentLength, target, endian);
}
if (hasSha1)
{
WriteContentLength(sha1ContentLength, target, endian);
}
long offsetToFirstFileInfo = (target.Position - startPosition).Align(1 << smallPackedAlignment);
StreamUtils.WriteZeros(target, offsetToFirstFileInfo - (target.Position - startPosition));
target.Position = startPosition;
WriteContentLength((uint)offsetToFirstFileInfo, target, endian);
target.Position = startPosition + offsetToFirstFileInfo;
long singleFileInfoLength = 16 + dummyContentLength + filenameLength + compressionInfoLength + bpsPatchInfoLength + crc32ContentLength + md5ContentLength + sha1ContentLength;
long totalFileInfoLength = singleFileInfoLength * files.Count;
long offsetToEndOfFileInfo = (offsetToFirstFileInfo + totalFileInfoLength).Align(1 << smallPackedAlignment);
StreamUtils.WriteZeros(target, offsetToEndOfFileInfo - offsetToFirstFileInfo);
var filedata = new List <(long position, DuplicatableStream data)>(files.Count);
long positionOfFreeSpace = offsetToEndOfFileInfo;
for (int i = 0; i < files.Count; ++i)
{
HyoutaArchiveFileInfo fi = files[i];
var fiData = fi.Data;
if (fiData == null)
{
throw new Exception("Data of file " + i + " is null.");
}
using (DuplicatableStream fs = fiData.Duplicate()) {
DuplicatableStream streamToWrite = fs;
bool streamIsInternallyCompressed = fi.StreamIsCompressed;
if (fi.BpsPatchInfo != null && fi.CompressionInfo != null && streamIsInternallyCompressed && !fi.StreamIsBpsPatch)
{
// this is a weird case; the stream wants both bps patch and compression
// and is already compressed but not already bps patched, which breaks the defined order
// we can handle this by decompressing, creating patch, recompressing
streamToWrite = fi.DataStream.Duplicate(); // this decompresses the stream
streamIsInternallyCompressed = false; // and fake-set the stream as uncompressed for packing logic
}
byte[]? bpsPatchInfoBytes = null;
byte[]? compressionInfoBytes = null;
if (hasBpsPatch)
{
if (fi.BpsPatchInfo == null)
{
// chunk has patches but this file is unpatched; we store this by pointing the file to itself
bpsPatchInfoBytes = new HyoutaArchiveBpsPatchInfo((ulong)i, (ulong)streamToWrite.Length, null).Serialize(endian);
}
else if (fi.StreamIsBpsPatch)
{
bpsPatchInfoBytes = fi.BpsPatchInfo.Serialize(endian);
}
else
{
var p = HyoutaArchiveBps.CreatePatch(fi.BpsPatchInfo, streamToWrite, endian);
bpsPatchInfoBytes = p.patchInfo;
streamToWrite = new DuplicatableByteArrayStream(p.patchData);
}
}
if (hasCompression && fi.CompressionInfo != null)
{
if (streamIsInternallyCompressed)
{
compressionInfoBytes = fi.CompressionInfo.Serialize(endian);
}
else
{
var p = fi.CompressionInfo.Compress(streamToWrite, endian);
compressionInfoBytes = p.compressionInfo;
streamToWrite = new DuplicatableByteArrayStream(p.compressedData);
}
}
// write file info
target.Position = (singleFileInfoLength * i) + offsetToFirstFileInfo + startPosition;
long positionPosition = target.Position;
target.WriteUInt64(0); // position of file, will be filled later
target.WriteUInt64((ulong)streamToWrite.Length, endian);
if (hasDummyContent)
{
if (fi.DummyContent != null)
{
target.Write(fi.DummyContent);
target.WriteZeros(dummyContentLength - fi.DummyContent.Length);
}
else
{
target.WriteZeros(dummyContentLength);
}
}
if (hasFilename)
{
if (fi.Filename != null)
{
var efn = encodedFilenames ![i];
public static (byte[] patchInfo, byte[] patchData) CreatePatch(HyoutaArchiveBpsPatchInfo patchInfo, Stream data, EndianUtils.Endianness endian)
{
throw new Exception("not yet implemented");
}
public HyoutaArchiveChunk(DuplicatableStream duplicatableStream, out ulong chunkLength)
{
using (DuplicatableStream data = duplicatableStream.Duplicate()) {
data.Position = 0;
// header
ulong extraMagic = data.ReadUInt64(EndianUtils.Endianness.LittleEndian);
ulong magic = extraMagic & 0x00fffffffffffffful;
if (magic != 0x6b6e7568636168)
{
throw new Exception("wrong magic");
}
byte extra = (byte)((extraMagic >> 56) & 0xffu);
byte packedAlignment = (byte)(extra & 0x1fu);
long unpackedAlignment = 1L << packedAlignment;
bool hasMetadata = (extra & 0x20) != 0;
bool isCompressed = (extra & 0x40) != 0;
bool isBigEndian = (extra & 0x80) != 0;
EndianUtils.Endianness e = isBigEndian ? EndianUtils.Endianness.BigEndian : EndianUtils.Endianness.LittleEndian;
ulong endOfFileOffset = data.ReadUInt64(e) << packedAlignment;
ulong tableOfContentsOffset = data.ReadUInt64(e) << packedAlignment;
ulong filecount = data.ReadUInt64(e);
chunkLength = endOfFileOffset;
if (hasMetadata)
{
// just skip past this for now
ulong metadataLength = data.ReadUInt64(e);
data.DiscardBytes(metadataLength);
}
DuplicatableStream dataBlockStream;
if (isCompressed)
{
ushort compressionInfoLengthRaw = data.ReadUInt16(e);
uint compressionInfoLength = compressionInfoLengthRaw & 0xfffcu;
int compressionInfoAlignmentPacked = (compressionInfoLengthRaw & 0x3) + 1;
data.ReadAlign(1u << compressionInfoAlignmentPacked);
Compression.IHyoutaArchiveCompressionInfo?compressionInfo = HyoutaArchiveCompression.Deserialize(data, compressionInfoLength == 0 ? 0x10000u : compressionInfoLength, e);
if (compressionInfo == null)
{
throw new Exception("File is indicated to be compressed, but no decompressor found.");
}
dataBlockStream = compressionInfo.Decompress(data);
}
else
{
data.ReadAlign(unpackedAlignment);
dataBlockStream = new PartialStream(data, data.Position, (long)(endOfFileOffset - (ulong)data.Position));
}
try {
data.Dispose();
dataBlockStream.Position = (long)tableOfContentsOffset;
uint offsetToFirstFileInfo = ReadContentLength(dataBlockStream, e);
// decode content bitfield(s)
long numberOfUnknownBits = 0;
ushort contentBitfield1 = dataBlockStream.ReadUInt16(e);
bool hasDummyContent = (contentBitfield1 & 0x0001u) != 0;
bool hasFilename = (contentBitfield1 & 0x0002u) != 0;
bool hasCompression = (contentBitfield1 & 0x0004u) != 0;
bool hasBpsPatch = (contentBitfield1 & 0x0008u) != 0;
bool hasCrc32 = (contentBitfield1 & 0x0010u) != 0;
bool hasMd5 = (contentBitfield1 & 0x0020u) != 0;
bool hasSha1 = (contentBitfield1 & 0x0040u) != 0;
numberOfUnknownBits += (contentBitfield1 & 0x0080u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0100u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0200u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0400u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x0800u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x1000u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x2000u) != 0 ? 1 : 0;
numberOfUnknownBits += (contentBitfield1 & 0x4000u) != 0 ? 1 : 0;
ushort currentBitfield = contentBitfield1;
while ((currentBitfield & 0x8000u) != 0)
{
// more bitfields, though we don't understand them since only the first handful of bits are defined at the moment, so just count and skip them
currentBitfield = dataBlockStream.ReadUInt16(e);
numberOfUnknownBits += (currentBitfield & 0x0001u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0002u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0004u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0008u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0010u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0020u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0040u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0080u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0100u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0200u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0400u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x0800u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x1000u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x2000u) != 0 ? 1 : 0;
numberOfUnknownBits += (currentBitfield & 0x4000u) != 0 ? 1 : 0;
}
uint dummyContentLength = hasDummyContent ? ReadContentLength(dataBlockStream, e) : 0;
uint filenameLength = hasFilename ? ReadContentLength(dataBlockStream, e) : 0;
uint compressionLength = hasCompression ? ReadContentLength(dataBlockStream, e) : 0;
uint bpspatchLength = hasBpsPatch ? ReadContentLength(dataBlockStream, e) : 0;
uint crc32Length = hasCrc32 ? ReadContentLength(dataBlockStream, e) : 0;
uint md5Length = hasMd5 ? ReadContentLength(dataBlockStream, e) : 0;
uint sha1Length = hasSha1 ? ReadContentLength(dataBlockStream, e) : 0;
long unknownContentLength = 0;
for (long i = 0; i < numberOfUnknownBits; ++i)
{
unknownContentLength += ReadContentLength(dataBlockStream, e);
}
dataBlockStream.Position = (long)(tableOfContentsOffset + offsetToFirstFileInfo);
List <HyoutaArchiveFileInfo> files = new List <HyoutaArchiveFileInfo>((int)filecount);
for (ulong i = 0; i < filecount; ++i)
{
ulong offset = dataBlockStream.ReadUInt64(e) << packedAlignment;
ulong filesize = dataBlockStream.ReadUInt64(e);
HyoutaArchiveFileInfo fi = new HyoutaArchiveFileInfo();
if (hasDummyContent)
{
fi.DummyContent = dataBlockStream.ReadBytes(dummyContentLength);
}
if (hasFilename)
{
fi.Filename = ReadString(dataBlockStream, filenameLength, e);
}
if (hasCompression)
{
fi.CompressionInfo = HyoutaArchiveCompression.Deserialize(dataBlockStream, compressionLength, e);
fi.StreamIsCompressed = true;
}
if (hasBpsPatch)
{
fi.BpsPatchInfo = HyoutaArchiveBpsPatchInfo.Deserialize(dataBlockStream, bpspatchLength, e, i, this);
fi.StreamIsBpsPatch = fi.BpsPatchInfo != null;
}
if (hasCrc32)
{
if (crc32Length >= 4)
{
fi.crc32 = new CRC32(dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian));
dataBlockStream.DiscardBytes(crc32Length - 4);
}
else
{
dataBlockStream.DiscardBytes(crc32Length);
}
}
if (hasMd5)
{
if (md5Length >= 16)
{
ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
fi.md5 = new MD5(a, b);
dataBlockStream.DiscardBytes(md5Length - 16);
}
else
{
dataBlockStream.DiscardBytes(md5Length);
}
}
if (hasSha1)
{
if (sha1Length >= 20)
{
ulong a = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
ulong b = dataBlockStream.ReadUInt64(EndianUtils.Endianness.BigEndian);
uint c = dataBlockStream.ReadUInt32(EndianUtils.Endianness.BigEndian);
fi.sha1 = new SHA1(a, b, c);
dataBlockStream.DiscardBytes(sha1Length - 20);
}
else
{
dataBlockStream.DiscardBytes(sha1Length);
}
}
dataBlockStream.DiscardBytes(unknownContentLength);
fi.Data = new PartialStream(dataBlockStream, (long)offset, (long)filesize);
files.Add(fi);
}
Files = files;
} finally {
dataBlockStream.Dispose();
}
}
}
