/// <summary>
/// Finds a valid entry for a given filename.
/// </summary>
/// <returns>The entry.</returns>
/// <param name="fileName">File name.</param>
public HashTableEntry FindEntry(string fileName)
{
uint entryHomeIndex = MPQCrypt.Hash(fileName, HashType.FileHashTableOffset) & (uint)this.Entries.Count - 1;
uint hashA = MPQCrypt.Hash(fileName, HashType.FilePathA);
uint hashB = MPQCrypt.Hash(fileName, HashType.FilePathB);
return(FindEntry(hashA, hashB, entryHomeIndex));
}
private List <uint> ReadFileSectorOffsetTable
(
[NotNull] BlockTableEntry fileBlockEntry,
uint fileKey = 0
)
{
var sectorOffsets = new List <uint>();
if (fileBlockEntry.IsEncrypted())
{
MPQCrypt.DecryptSectorOffsetTable(_archiveReader, ref sectorOffsets, fileBlockEntry.GetBlockSize(), fileKey - 1);
}
else
{
// As protection against corrupt or maliciously zeroed blocks or corrupt blocks,
// reading will be escaped early if the sector offset table is not consistent.
// Should the total size as predicted by the sector offset table go beyond the total
// block size, or if an offset is not unique, no file will be read and the function will
// escape early.
uint sectorOffset = 0;
while (sectorOffset != fileBlockEntry.GetBlockSize())
{
sectorOffset = _archiveReader.ReadUInt32();
// Should the resulting sector offset be less than the previous data, then the data is inconsistent
// and no table should be returned.
if (sectorOffsets.LastOrDefault() > sectorOffset)
{
throw new InvalidFileSectorTableException(
"The read offset in the sector table was less than the previous offset.");
}
// Should the resulting sector offset be greater than the total block size, then the data is
// inconsistent and no file should be returned.
if (sectorOffset > fileBlockEntry.GetBlockSize())
{
throw new InvalidFileSectorTableException(
"The read offset in the sector table was greater than the total size of the data block.");
}
// Should the resulting sector not be unique, something is wrong and no table should be returned.
if (sectorOffsets.Contains(sectorOffset))
{
throw new InvalidFileSectorTableException(
"The read offset in the sector table was not unique to the whole table.");
}
// The offset should be valid, so add it to the table.
sectorOffsets.Add(sectorOffset);
}
}
return(sectorOffsets);
}
public bool TryFindEntry([NotNull] string fileName, out HashTableEntry tableEntry)
{
var entryHomeIndex = MPQCrypt.Hash(fileName, HashType.FileHashTableOffset) & (uint)_entries.Count - 1;
var hashA = MPQCrypt.Hash(fileName, HashType.FilePathA);
var hashB = MPQCrypt.Hash(fileName, HashType.FilePathB);
if (!TryFindEntry(hashA, hashB, entryHomeIndex, out tableEntry))
{
return(false);
}
return(true);
}
/// <summary>
/// Serializes the current object into a byte array.
/// </summary>
/// <inheritdoc/>
public byte[] Serialize()
{
using var ms = new MemoryStream();
using (var bw = new BinaryWriter(ms))
{
foreach (var entry in _entries)
{
bw.Write(entry.Serialize());
}
}
var encryptedTable = MPQCrypt.EncryptData(ms.ToArray(), TableKey);
return(encryptedTable);
}
/// <inheritdoc/>
public byte[] Serialize()
{
using (MemoryStream ms = new MemoryStream())
{
using (BinaryWriter bw = new BinaryWriter(ms))
{
foreach (BlockTableEntry entry in Entries)
{
bw.Write(entry.Serialize());
}
}
byte[] encryptedTable = MPQCrypt.EncryptData(ms.ToArray(), TableKey);
return(encryptedTable);
}
}
private byte[] ExtractUncompressedSectoredFile
(
[NotNull] BlockTableEntry fileBlockEntry,
uint fileKey
)
{
// This file uses sectoring, but is not compressed. It may be encrypted.
var finalSectorSize = fileBlockEntry.GetFileSize() % GetMaxSectorSize();
// All the even sectors you can fit into the file size
var sectorCount = (fileBlockEntry.GetFileSize() - finalSectorSize) / GetMaxSectorSize();
var rawSectors = new List <byte[]>();
for (var i = 0; i < sectorCount; ++i)
{
// Read a normal sector (usually 4096 bytes)
rawSectors.Add(_archiveReader.ReadBytes((int)GetMaxSectorSize()));
}
// And finally, if there's an uneven sector at the end, read that one too
if (finalSectorSize > 0)
{
rawSectors.Add(_archiveReader.ReadBytes((int)finalSectorSize));
}
uint sectorIndex = 0;
var finalSectors = new List <byte[]>();
foreach (var rawSector in rawSectors)
{
var pendingSector = rawSector;
if (fileBlockEntry.IsEncrypted())
{
// Decrypt the block
pendingSector = MPQCrypt.DecryptData(rawSector, fileKey + sectorIndex);
}
finalSectors.Add(pendingSector);
++sectorIndex;
}
return(StitchSectors(finalSectors));
}
/// <summary>
/// Extracts a file which is stored as a single unit in the archive.
/// </summary>
/// <param name="fileBlockEntry">The block entry of the file.</param>
/// <param name="fileKey">The encryption key of the file.</param>
/// <returns>The complete file data.</returns>
private byte[] ExtractSingleUnitFile(BlockTableEntry fileBlockEntry, uint fileKey)
{
// This file does not use sectoring, but may still be encrypted or compressed.
byte[] fileData = this.ArchiveReader.ReadBytes((int)fileBlockEntry.GetBlockSize());
if (fileBlockEntry.IsEncrypted())
{
// Decrypt the block
fileData = MPQCrypt.DecryptData(fileData, fileKey);
}
// Decompress the sector if neccesary
if (fileBlockEntry.IsCompressed())
{
fileData = Compression.DecompressSector(fileData, fileBlockEntry.Flags);
}
return(fileData);
}
/// <summary>
/// Initializes a new instance of the <see cref="Warcraft.MPQ.MPQ"/> class.
/// </summary>
/// <param name="mpqStream">An open stream to data containing an MPQ archive.</param>
public MPQ([NotNull] Stream mpqStream)
{
_archiveReader = new BinaryReader(mpqStream);
Header = new MPQHeader(_archiveReader.ReadBytes((int)PeekHeaderSize()));
// Seek to the hash table and load it
_archiveReader.BaseStream.Position = (long)Header.GetHashTableOffset();
var encryptedHashTable = _archiveReader.ReadBytes((int)Header.GetHashTableSize());
var hashTableData = MPQCrypt.DecryptData(encryptedHashTable, HashTable.TableKey);
ArchiveHashTable = new HashTable(hashTableData);
// Seek to the block table and load it
_archiveReader.BaseStream.Position = (long)Header.GetBlockTableOffset();
var encryptedBlockTable = _archiveReader.ReadBytes((int)Header.GetBlockTableSize());
var blockTableData = MPQCrypt.DecryptData(encryptedBlockTable, BlockTable.TableKey);
ArchiveBlockTable = new BlockTable(blockTableData);
if (Header.GetFormat() != MPQFormat.ExtendedV1)
{
return;
}
// Seek to the extended block table and load it, if necessary
if (Header.GetExtendedBlockTableOffset() <= 0)
{
return;
}
_archiveReader.BaseStream.Position = (long)Header.GetExtendedBlockTableOffset();
var extendedTable = new List <ushort>();
for (var i = 0; i < Header.GetBlockTableEntryCount(); ++i)
{
extendedTable.Add(_archiveReader.ReadUInt16());
}
ExtendedBlockTable = extendedTable;
}
/// <summary>
/// Extracts a file which is divided into a set of compressed sectors.
/// </summary>
/// <param name="fileBlockEntry">The block entry of the file.</param>
/// <param name="fileKey">The encryption key of the file.</param>
/// <param name="adjustedBlockOffset">The offset to where the file sectors begin.</param>
/// <returns>The complete file data.</returns>
/// <exception cref="InvalidFileSectorTableException">Thrown if the sector table is found to be inconsistent in any way.</exception>
private byte[] ExtractCompressedSectoredFile(BlockTableEntry fileBlockEntry, uint fileKey, long adjustedBlockOffset)
{
// This file uses sectoring, and is compressed. It may be encrypted.
//Retrieve the offsets for each sector - these are relative to the beginning of the data.
List <uint> sectorOffsets = ReadFileSectorOffsetTable(fileBlockEntry, fileKey);
// Read all of the raw file sectors.
List <byte[]> compressedSectors = new List <byte[]>();
for (int i = 0; i < sectorOffsets.Count - 1; ++i)
{
long sectorStartPosition = adjustedBlockOffset + sectorOffsets[i];
this.ArchiveReader.BaseStream.Position = sectorStartPosition;
uint sectorLength = sectorOffsets[i + 1] - sectorOffsets[i];
compressedSectors.Add(this.ArchiveReader.ReadBytes((int)sectorLength));
}
// Begin decompressing and decrypting the sectors
// TODO: If Checksums are present (check the flags), treat the last sector as a checksum sector
// TODO: Check "backup.MPQ/realmlist.wtf" for a weird file with checksums that is not working correctly.
// It has a single sector with a single checksum after it, and none of the hashing functions seem to
// produce a valid hash. CRC32, Adler32, CRC32B, nothing.
// Some flags (listfiles mostly) are flagged as having checksums but don't have a checksum sector.
// Perhaps related to attributes file?
List <byte[]> decompressedSectors = new List <byte[]>();
/* List<uint> SectorChecksums = new List<uint>();
* if (fileBlockEntry.Flags.HasFlag(BlockFlags.BLF_HasChecksums))
* {
* byte[] compressedChecksums = compressedSectors.Last();
* byte[] decompressedChecksums = Compression.DecompressSector(compressedChecksums, fileBlockEntry.Flags);
*
* // Lift out the last sector and treat it as a checksum sector
* using (MemoryStream ms = new MemoryStream(decompressedChecksums))
* {
* using (BinaryReader br = new BinaryReader(ms))
* {
* // Drop the checksum sector from the file sectors
* compressedSectors.RemoveAt(compressedSectors.Count - 1);
*
* for (int i = 0; i < compressedSectors.Count; ++i)
* {
* SectorChecksums.Add(br.ReadUInt32());
* }
* }
* }
* }*/
uint sectorIndex = 0;
foreach (byte[] compressedSector in compressedSectors)
{
byte[] pendingSector = compressedSector;
if (fileBlockEntry.IsEncrypted())
{
// Decrypt the block
pendingSector = MPQCrypt.DecryptData(compressedSector, fileKey + sectorIndex);
}
/*if (fileBlockEntry.Flags.HasFlag(BlockFlags.HasCRCChecksums))
* {
* // Verify the sector
* bool isSectorIntact = MPQCrypt.VerifySectorChecksum(pendingSector, SectorChecksums[(int)sectorIndex]);
* if (!isSectorIntact)
* {
* using (MemoryStream ms = new MemoryStream(pendingSector))
* {
* //DEBUG
*
* uint sectorChecksum = (uint)Adler32.ComputeChecksum(ms);
*
* string exceptionMessage = String.Format("The decrypted sector failed its integrity checking. \n" +
* "The sector had a checksum of \"{0}\", and the expected one was \"{1}\".",
* sectorChecksum, SectorChecksums[(int)sectorIndex]);
*
* throw new InvalidDataException(exceptionMessage);
* }
* }
* }*/
// Decompress the sector if neccesary
if (pendingSector.Length < GetMaxSectorSize())
{
int currentFileSize = CountBytesInSectors(decompressedSectors);
bool canSectorCompleteFile = currentFileSize + pendingSector.Length == fileBlockEntry.GetFileSize();
if (!canSectorCompleteFile && currentFileSize != fileBlockEntry.GetFileSize())
{
pendingSector = Compression.DecompressSector(pendingSector, fileBlockEntry.Flags);
}
}
decompressedSectors.Add(pendingSector);
++sectorIndex;
}
return(StitchSectors(decompressedSectors));
}
// TODO: Filter files based on language and platform
/// <summary>
/// Extract the file at <paramref name="filePath"/> from the archive.
/// </summary>
/// <returns>The file as a byte array, or null if the file could not be found.</returns>
/// <param name="filePath">Path to the file in the archive.</param>
public byte[] ExtractFile(string filePath)
{
if (this.IsDisposed)
{
throw new ObjectDisposedException(ToString(), "Cannot use a disposed archive.");
}
// Reset all positions to be safe
this.ArchiveReader.BaseStream.Position = 0;
HashTableEntry fileHashEntry = this.ArchiveHashTable.FindEntry(filePath);
if (fileHashEntry == null)
{
return(null);
}
BlockTableEntry fileBlockEntry = this.ArchiveBlockTable.GetEntry((int)fileHashEntry.GetBlockEntryIndex());
// Drop out if the file is not actually a file
if (!fileBlockEntry.HasData())
{
return(null);
}
// Seek to the beginning of the file's sectors
long adjustedBlockOffset;
if (this.Header.GetFormat() >= MPQFormat.ExtendedV1 && RequiresExtendedFormat())
{
ushort upperOffsetBits = this.ExtendedBlockTable[(int)fileHashEntry.GetBlockEntryIndex()];
adjustedBlockOffset = (long)fileBlockEntry.GetExtendedBlockOffset(upperOffsetBits);
}
else
{
adjustedBlockOffset = fileBlockEntry.GetBlockOffset();
}
this.ArchiveReader.BaseStream.Position = adjustedBlockOffset;
// Calculate the decryption key if neccesary
uint fileKey = MPQCrypt.CreateFileEncryptionKey
(
filePath,
fileBlockEntry.ShouldEncryptionKeyBeAdjusted(),
adjustedBlockOffset,
fileBlockEntry.GetFileSize()
);
// Examine the file storage types and extract as neccesary
if (fileBlockEntry.IsSingleUnit())
{
return(ExtractSingleUnitFile(fileBlockEntry, fileKey));
}
if (fileBlockEntry.IsCompressed())
{
return(ExtractCompressedSectoredFile(fileBlockEntry, fileKey, adjustedBlockOffset));
}
return(ExtractUncompressedSectoredFile(fileBlockEntry, fileKey));
}
/// <summary>
/// Initializes a new instance of the <see cref="Warcraft.MPQ.MPQ"/> class.
/// </summary>
/// <param name="mpqStream">An open stream to data containing an MPQ archive.</param>
public MPQ(Stream mpqStream)
{
this.ArchiveReader = new BinaryReader(mpqStream);
this.Header = new MPQHeader(this.ArchiveReader.ReadBytes((int)PeekHeaderSize()));
// Seek to the hash table and load it
this.ArchiveReader.BaseStream.Position = (long)this.Header.GetHashTableOffset();
byte[] hashTableData;
if (this.Header.IsHashTableCompressed())
{
byte[] encryptedData = this.ArchiveReader.ReadBytes((int)this.Header.GetCompressedHashTableSize());
byte[] decryptedData = MPQCrypt.DecryptData(encryptedData, HashTable.TableKey);
BlockFlags tableFlags = BlockFlags.IsCompressedMultiple;
hashTableData = Compression.DecompressSector(decryptedData, tableFlags);
}
else
{
byte[] encryptedData = this.ArchiveReader.ReadBytes((int)this.Header.GetHashTableSize());
hashTableData = MPQCrypt.DecryptData(encryptedData, HashTable.TableKey);
}
this.ArchiveHashTable = new HashTable(hashTableData);
// Seek to the block table and load it
this.ArchiveReader.BaseStream.Position = (long)this.Header.GetBlockTableOffset();
byte[] blockTableData;
if (this.Header.IsBlockTableCompressed())
{
byte[] encryptedData = this.ArchiveReader.ReadBytes((int)this.Header.GetCompressedBlockTableSize());
byte[] decryptedData = MPQCrypt.DecryptData(encryptedData, BlockTable.TableKey);
BlockFlags tableFlags = BlockFlags.IsCompressedMultiple;
blockTableData = Compression.DecompressSector(decryptedData, tableFlags);
}
else
{
byte[] encryptedData = this.ArchiveReader.ReadBytes((int)this.Header.GetBlockTableSize());
blockTableData = MPQCrypt.DecryptData(encryptedData, BlockTable.TableKey);
}
this.ArchiveBlockTable = new BlockTable(blockTableData);
// TODO: Seek to the extended hash table and load it
// TODO: Seek to the extended block table and load it
if (this.Header.GetFormat() >= MPQFormat.ExtendedV1)
{
// Seek to the extended block table and load it, if neccesary
if (this.Header.GetExtendedBlockTableOffset() > 0)
{
this.ArchiveReader.BaseStream.Position = (long)this.Header.GetExtendedBlockTableOffset();
for (int i = 0; i < this.Header.GetBlockTableEntryCount(); ++i)
{
this.ExtendedBlockTable.Add(this.ArchiveReader.ReadUInt16());
}
}
}
}
/// <inheritdoc />
/// <exception cref="ObjectDisposedException">Thrown if the archive has been disposed.</exception>
/// <exception cref="FileNotFoundException">Thrown if the archive does not contain a file at the given path.</exception>
/// <exception cref="FileDeletedException">Thrown if the file is deleted in the archive.</exception>
public byte[] ExtractFile(string filePath)
{
ThrowIfDisposed();
// Reset all positions to be safe
ArchiveReader.BaseStream.Position = 0;
HashTableEntry fileHashEntry;
try
{
fileHashEntry = ArchiveHashTable.FindEntry(filePath);
}
catch (FileNotFoundException fex)
{
throw new FileNotFoundException("No file found at the given path.", filePath, fex);
}
BlockTableEntry fileBlockEntry = ArchiveBlockTable.GetEntry((int)fileHashEntry.GetBlockEntryIndex());
// Drop out if the file has been deleted
if (fileBlockEntry.IsDeleted())
{
throw new FileDeletedException("The given file is deleted.", filePath);
}
// Seek to the beginning of the file's sectors
long adjustedBlockOffset;
if (Header.GetFormat() >= MPQFormat.ExtendedV1 && RequiresExtendedFormat())
{
ushort upperOffsetBits = ExtendedBlockTable[(int)fileHashEntry.GetBlockEntryIndex()];
adjustedBlockOffset = (long)fileBlockEntry.GetExtendedBlockOffset(upperOffsetBits);
}
else
{
adjustedBlockOffset = fileBlockEntry.GetBlockOffset();
}
ArchiveReader.BaseStream.Position = adjustedBlockOffset;
// Calculate the decryption key if neccesary
uint fileKey = MPQCrypt.CreateFileEncryptionKey
(
filePath,
fileBlockEntry.ShouldEncryptionKeyBeAdjusted(),
adjustedBlockOffset,
fileBlockEntry.GetFileSize()
);
// Examine the file storage types and extract as neccesary
if (fileBlockEntry.IsSingleUnit())
{
return(ExtractSingleUnitFile(fileBlockEntry, fileKey));
}
if (fileBlockEntry.IsCompressed())
{
return(ExtractCompressedSectoredFile(fileBlockEntry, fileKey, adjustedBlockOffset));
}
return(ExtractUncompressedSectoredFile(fileBlockEntry, fileKey));
}
public bool TryExtractFile(string filePath, out byte[] data)
{
ThrowIfDisposed();
data = null;
// Reset all positions to be safe
_archiveReader.BaseStream.Position = 0;
if (!ArchiveHashTable.TryFindEntry(filePath, out var fileHashEntry))
{
return(false);
}
var fileBlockEntry = ArchiveBlockTable.GetEntry((int)fileHashEntry.GetBlockEntryIndex());
// Drop out if the file has been deleted
if (fileBlockEntry.IsDeleted())
{
return(false);
}
// Seek to the beginning of the file's sectors
long adjustedBlockOffset;
if (Header.GetFormat() == MPQFormat.ExtendedV1 && RequiresExtendedFormat())
{
var upperOffsetBits = ExtendedBlockTable[(int)fileHashEntry.GetBlockEntryIndex()];
adjustedBlockOffset = (long)fileBlockEntry.GetExtendedBlockOffset(upperOffsetBits);
}
else
{
adjustedBlockOffset = fileBlockEntry.GetBlockOffset();
}
_archiveReader.BaseStream.Position = adjustedBlockOffset;
// Calculate the decryption key if necessary
var fileKey = MPQCrypt.CreateFileEncryptionKey
(
filePath,
fileBlockEntry.ShouldEncryptionKeyBeAdjusted(),
adjustedBlockOffset,
fileBlockEntry.GetFileSize()
);
// Examine the file storage types and extract as necessary
if (fileBlockEntry.IsSingleUnit())
{
data = ExtractSingleUnitFile(fileBlockEntry, fileKey);
return(true);
}
if (fileBlockEntry.IsCompressed())
{
data = ExtractCompressedSectoredFile(fileBlockEntry, fileKey, adjustedBlockOffset);
return(true);
}
data = ExtractUncompressedSectoredFile(fileBlockEntry, fileKey);
return(true);
}