void ReadIndexInternal(byte[] key, PakFilter filter, out Exception exc)
{
if (Initialized)
{
exc = new InvalidOperationException("Index is already initialized");
return;
}
if (Info.bEncryptedIndex && key == null)
{
exc = new ArgumentException("Index is encrypted but no key was provided", nameof(key));
return;
}
Stream.Position = Info.IndexOffset;
BinaryReader IndexReader;
if (Info.bEncryptedIndex)
{
IndexReader = new BinaryReader(new MemoryStream(AESDecryptor.DecryptAES(Reader.ReadBytes((int)Info.IndexSize), key)));
int stringLen = IndexReader.ReadInt32();
if (stringLen > 512 || stringLen < -512)
{
exc = new ArgumentException("The provided key is invalid", nameof(key));
return;
}
if (stringLen < 0)
{
IndexReader.BaseStream.Position += (stringLen - 1) * 2;
if (IndexReader.ReadUInt16() != 0)
{
exc = new ArgumentException("The provided key is invalid", nameof(key));
return;
}
}
else
{
IndexReader.BaseStream.Position += stringLen - 1;
if (IndexReader.ReadByte() != 0)
{
exc = new ArgumentException("The provided key is invalid", nameof(key));
return;
}
}
IndexReader.BaseStream.Position = 0;
}
else
{
IndexReader = Reader;
}
Dictionary <string, FPakEntry> tempFiles;
if (Info.Version >= EPakVersion.PATH_HASH_INDEX)
{
ReadIndexUpdated(IndexReader, key, out tempFiles, filter);
}
else
{
// https://github.com/EpicGames/UnrealEngine/blob/bf95c2cbc703123e08ab54e3ceccdd47e48d224a/Engine/Source/Runtime/PakFile/Private/IPlatformFilePak.cpp#L4509
MountPoint = IndexReader.ReadFString();
if (MountPoint.StartsWith("../../.."))
{
MountPoint = MountPoint.Substring(8);
}
else
{
// Weird mount point location...
MountPoint = "/";
}
if (!CaseSensitive)
{
MountPoint = MountPoint.ToLowerInvariant();
}
var NumEntries = IndexReader.ReadInt32();
tempFiles = new Dictionary <string, FPakEntry>(NumEntries);
for (int i = 0; i < NumEntries; i++)
{
var entry = new FPakEntry(IndexReader, Info.Version, CaseSensitive, FileName);
// if there is no filter OR the filter passes
if (filter == null || filter.CheckFilter(MountPoint + entry.Name, CaseSensitive))
{
// Filename is without the MountPoint concatenated to save memory
tempFiles[entry.Name] = entry;
}
}
}
Paks.Merge(tempFiles, out var files, MountPoint);
Entries = files;
DebugHelper.WriteLine("{0} {1} {2} {3}", "[FModel]", "[PakFileReader]", "[ReadIndexInternal]", $"{FileName} contains {Entries.Count} files, mount point: \"{this.MountPoint}\", version: {(int)this.Info.Version}");
if (Info.bEncryptedIndex)
{
// underlying stream is a MemoryStream of the decrypted index, might improve performance with a crypto stream of some sort
IndexReader.Dispose();
}
Reader.Dispose();
Initialized = true;
exc = null;
}
void ReadIndexUpdated(BinaryReader reader, byte[] aesKey, out Dictionary <string, FPakEntry> dict, PakFilter filter)
{
MountPoint = reader.ReadFString();
if (MountPoint.StartsWith("../../.."))
{
MountPoint = MountPoint.Substring(8);
}
else
{
// Weird mount point location...
MountPoint = "/";
}
if (!CaseSensitive)
{
MountPoint = MountPoint.ToLowerInvariant();
}
var NumEntries = reader.ReadInt32();
var PathHashSeed = reader.ReadUInt64();
if (reader.ReadInt32() == 0)
{
throw new FileLoadException("No path hash index");
}
/*
* long PathHashIndexOffset = reader.ReadInt64();
* long PathHashIndexSize = reader.ReadInt64();
* FSHAHash PathHashIndexHash = new FSHAHash(reader);
*/
reader.BaseStream.Position += 8L + 8L + 20L;
if (reader.ReadInt32() == 0)
{
throw new FileLoadException("No directory index");
}
long FullDirectoryIndexOffset = reader.ReadInt64();
long FullDirectoryIndexSize = reader.ReadInt64();
FSHAHash FullDirectoryIndexHash = new FSHAHash(reader);
byte[] EncodedPakEntries = reader.ReadTArray(() => reader.ReadByte());
int FilesNum = reader.ReadInt32();
if (FilesNum < 0)
{
// Should not be possible for any values in the PrimaryIndex to be invalid, since we verified the index hash
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
}
Reader.BaseStream.Position = FullDirectoryIndexOffset;
byte[] PathHashIndexData = Reader.ReadBytes((int)FullDirectoryIndexSize);
if (!DecryptAndValidateIndex(ref PathHashIndexData, aesKey, FullDirectoryIndexHash, out var ComputedHash))
{
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
//UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
//UE_LOG(LogPakFile, Log, TEXT(" Encrypted: %d"), Info.bEncryptedIndex);
//UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
//UE_LOG(LogPakFile, Log, TEXT(" Index Offset: %d"), FullDirectoryIndexOffset);
//UE_LOG(LogPakFile, Log, TEXT(" Index Size: %d"), FullDirectoryIndexSize);
//UE_LOG(LogPakFile, Log, TEXT(" Stored Index Hash: %s"), *PathHashIndexHash.ToString());
//UE_LOG(LogPakFile, Log, TEXT(" Computed Index Hash: %s"), *ComputedHash.ToString());
}
BinaryReader PathHashIndexReader = new BinaryReader(new MemoryStream(PathHashIndexData));
FPakDirectoryEntry[] PathHashIndex = PathHashIndexReader.ReadTArray(() => new FPakDirectoryEntry(PathHashIndexReader));
dict = new Dictionary <string, FPakEntry>(NumEntries);
foreach (FPakDirectoryEntry directoryEntry in PathHashIndex)
{
foreach (FPathHashIndexEntry hashIndexEntry in directoryEntry.Entries)
{
var path = directoryEntry.Directory + hashIndexEntry.Filename;
if (path.StartsWith("/"))
{
path = path.Substring(1);
}
if (!CaseSensitive)
{
path = path.ToLowerInvariant();
}
// if there is no filter OR the filter passes
if (filter == null || filter.CheckFilter(MountPoint + hashIndexEntry.Filename, CaseSensitive))
{
// Filename is without the MountPoint concatenated to save memory
dict[path] = GetEntry(path, hashIndexEntry.Location, EncodedPakEntries);
}
}
}
}
void ReadIndexUpdated(BinaryReader reader, byte[] aesKey, long totalSize, PakFilter filter)
{
MountPoint = reader.ReadFString();
if (MountPoint.StartsWith("../../.."))
{
MountPoint = MountPoint.Substring(8);
}
else
{
// Weird mount point location...
MountPoint = "/";
}
if (!CaseSensitive)
{
MountPoint = MountPoint.ToLowerInvariant();
}
var NumEntries = reader.ReadInt32();
var PathHashSeed = reader.ReadUInt64();
bool bReaderHasPathHashIndex = false;
long PathHashIndexOffset = -1; // INDEX_NONE
long PathHashIndexSize = 0;
FSHAHash PathHashIndexHash = default;
bReaderHasPathHashIndex = reader.ReadInt32() != 0;
if (bReaderHasPathHashIndex)
{
PathHashIndexOffset = reader.ReadInt64();
PathHashIndexSize = reader.ReadInt64();
PathHashIndexHash = new FSHAHash(reader);
bReaderHasPathHashIndex = bReaderHasPathHashIndex && PathHashIndexOffset != -1;
}
bool bReaderHasFullDirectoryIndex = false;
long FullDirectoryIndexOffset = -1; // INDEX_NONE
long FullDirectoryIndexSize = 0;
FSHAHash FullDirectoryIndexHash = default;
bReaderHasFullDirectoryIndex = reader.ReadInt32() != 0;
if (bReaderHasFullDirectoryIndex)
{
FullDirectoryIndexOffset = reader.ReadInt64();
FullDirectoryIndexSize = reader.ReadInt64();
FullDirectoryIndexHash = new FSHAHash(reader);
bReaderHasFullDirectoryIndex = bReaderHasFullDirectoryIndex && FullDirectoryIndexOffset != -1;
}
byte[] EncodedPakEntries = reader.ReadTArray(() => reader.ReadByte());
File.WriteAllBytes("pakentryencoded", EncodedPakEntries);
int FilesNum = reader.ReadInt32();
if (FilesNum < 0)
{
// Should not be possible for any values in the PrimaryIndex to be invalid, since we verified the index hash
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
}
FPakEntry[] Files = new FPakEntry[FilesNum]; // from what i can see, there aren't any???
if (FilesNum > 0)
{
for (int FileIndex = 0; FileIndex < FilesNum; ++FileIndex)
{
Files[FileIndex] = new FPakEntry(reader, Info.Version);
}
}
// Decide which SecondaryIndex(es) to load
bool bWillUseFullDirectoryIndex;
bool bWillUsePathHashIndex;
bool bReadFullDirectoryIndex;
if (bReaderHasPathHashIndex && bReaderHasFullDirectoryIndex)
{
bWillUseFullDirectoryIndex = false; // https://github.com/EpicGames/UnrealEngine/blob/79a64829237ae339118bb50b61d84e4599c14e8a/Engine/Source/Runtime/PakFile/Private/IPlatformFilePak.cpp#L5628
bWillUsePathHashIndex = !bWillUseFullDirectoryIndex;
bool bWantToReadFullDirectoryIndex = false;
bReadFullDirectoryIndex = bReaderHasFullDirectoryIndex && bWantToReadFullDirectoryIndex;
}
else if (bReaderHasPathHashIndex)
{
bWillUsePathHashIndex = true;
bWillUseFullDirectoryIndex = false;
bReadFullDirectoryIndex = false;
}
else if (bReaderHasFullDirectoryIndex)
{
// We don't support creating the PathHash Index at runtime; we want to move to having only the PathHashIndex, so supporting not having it at all is not useful enough to write
bWillUsePathHashIndex = false;
bWillUseFullDirectoryIndex = true;
bReadFullDirectoryIndex = true;
}
else
{
// It should not be possible for PrimaryIndexes to be built without a PathHashIndex AND without a FullDirectoryIndex; CreatePakFile in UnrealPak.exe has a check statement for it.
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
}
// Load the Secondary Index(es)
byte[] PathHashIndexData;
Dictionary <ulong, int> PathHashIndex;
BinaryReader PathHashIndexReader = default;
bool bHasPathHashIndex;
if (bWillUsePathHashIndex)
{
if (PathHashIndexOffset < 0 || totalSize < (PathHashIndexOffset + PathHashIndexSize))
{
// Should not be possible for these values (which came from the PrimaryIndex) to be invalid, since we verified the index hash of the PrimaryIndex
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
//UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
//UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
//UE_LOG(LogPakFile, Log, TEXT(" PathHashIndexOffset : %d"), PathHashIndexOffset);
//UE_LOG(LogPakFile, Log, TEXT(" PathHashIndexSize: %d"), PathHashIndexSize);
}
Reader.BaseStream.Position = PathHashIndexOffset;
PathHashIndexData = Reader.ReadBytes((int)PathHashIndexSize);
File.WriteAllBytes("indexdata.daa", PathHashIndexData);
{
if (!DecryptAndValidateIndex(Reader, ref PathHashIndexData, aesKey, PathHashIndexHash, out var ComputedHash))
{
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
//UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
//UE_LOG(LogPakFile, Log, TEXT(" Encrypted: %d"), Info.bEncryptedIndex);
//UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
//UE_LOG(LogPakFile, Log, TEXT(" Index Offset: %d"), FullDirectoryIndexOffset);
//UE_LOG(LogPakFile, Log, TEXT(" Index Size: %d"), FullDirectoryIndexSize);
//UE_LOG(LogPakFile, Log, TEXT(" Stored Index Hash: %s"), *PathHashIndexHash.ToString());
//UE_LOG(LogPakFile, Log, TEXT(" Computed Index Hash: %s"), *ComputedHash.ToString());
}
}
PathHashIndexReader = new BinaryReader(new MemoryStream(PathHashIndexData));
PathHashIndex = ReadPathHashIndex(PathHashIndexReader);
bHasPathHashIndex = true;
}
var DirectoryIndex = new Dictionary <string, Dictionary <string, int> >();
bool bHasFullDirectoryIndex;
if (!bReadFullDirectoryIndex)
{
DirectoryIndex = ReadDirectoryIndex(PathHashIndexReader);
bHasFullDirectoryIndex = false;
}
if (DirectoryIndex.Count == 0)
{
if (totalSize < (FullDirectoryIndexOffset + FullDirectoryIndexSize) ||
FullDirectoryIndexOffset < 0)
{
// Should not be possible for these values (which came from the PrimaryIndex) to be invalid, since we verified the index hash of the PrimaryIndex
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
//UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
//UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
//UE_LOG(LogPakFile, Log, TEXT(" FullDirectoryIndexOffset : %d"), FullDirectoryIndexOffset);
//UE_LOG(LogPakFile, Log, TEXT(" FullDirectoryIndexSize: %d"), FullDirectoryIndexSize);
}
Reader.BaseStream.Position = FullDirectoryIndexOffset;
byte[] FullDirectoryIndexData = Reader.ReadBytes((int)FullDirectoryIndexSize);
{
if (!DecryptAndValidateIndex(Reader, ref FullDirectoryIndexData, aesKey, FullDirectoryIndexHash, out var ComputedHash))
{
throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
//UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
//UE_LOG(LogPakFile, Log, TEXT(" Encrypted: %d"), Info.bEncryptedIndex);
//UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
//UE_LOG(LogPakFile, Log, TEXT(" Index Offset: %d"), FullDirectoryIndexOffset);
//UE_LOG(LogPakFile, Log, TEXT(" Index Size: %d"), FullDirectoryIndexSize);
//UE_LOG(LogPakFile, Log, TEXT(" Stored Index Hash: %s"), *FullDirectoryIndexHash.ToString());
//UE_LOG(LogPakFile, Log, TEXT(" Computed Index Hash: %s"), *ComputedHash.ToString());
}
}
var SecondaryIndexReader = new BinaryReader(new MemoryStream(FullDirectoryIndexData));
DirectoryIndex = ReadDirectoryIndex(SecondaryIndexReader);
bHasFullDirectoryIndex = true;
}
Entries = new Dictionary <string, FPakEntry>(NumEntries);
foreach (var(dirname, dir) in DirectoryIndex)
{
foreach (var(filename, pakLocation) in dir)
{
var path = dirname + filename;
if (!CaseSensitive)
{
path = path.ToLowerInvariant();
}
// if there is no filter OR the filter passes
if (filter == null || filter.CheckFilter(MountPoint + filename, CaseSensitive))
{
// Filename is without the MountPoint concatenated to save memory
Entries[path] = GetEntry(pakLocation, EncodedPakEntries);
}
}
}
}