Example #1
0
        void ReadIndexUpdated(BinaryReader reader, byte[] aesKey, out Dictionary <string, FPakEntry> dict, PakFilter filter)
        {
            MountPoint = reader.ReadFString();
            if (MountPoint.StartsWith("../../.."))
            {
                MountPoint = MountPoint.Substring(8);
            }
            else
            {
                // Weird mount point location...
                MountPoint = "/";
            }
            if (!CaseSensitive)
            {
                MountPoint = MountPoint.ToLowerInvariant();
            }
            var NumEntries   = reader.ReadInt32();
            var PathHashSeed = reader.ReadUInt64();

            if (reader.ReadInt32() == 0)
            {
                throw new FileLoadException("No path hash index");
            }

            /*
             * long PathHashIndexOffset = reader.ReadInt64();
             * long PathHashIndexSize = reader.ReadInt64();
             * FSHAHash PathHashIndexHash = new FSHAHash(reader);
             */
            reader.BaseStream.Position += 8L + 8L + 20L;

            if (reader.ReadInt32() == 0)
            {
                throw new FileLoadException("No directory index");
            }

            long     FullDirectoryIndexOffset = reader.ReadInt64();
            long     FullDirectoryIndexSize   = reader.ReadInt64();
            FSHAHash FullDirectoryIndexHash   = new FSHAHash(reader);

            byte[] EncodedPakEntries = reader.ReadTArray(() => reader.ReadByte());

            int FilesNum = reader.ReadInt32();

            if (FilesNum < 0)
            {
                // Should not be possible for any values in the PrimaryIndex to be invalid, since we verified the index hash
                throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
            }

            Reader.BaseStream.Position = FullDirectoryIndexOffset;
            byte[] PathHashIndexData = Reader.ReadBytes((int)FullDirectoryIndexSize);

            if (!DecryptAndValidateIndex(ref PathHashIndexData, aesKey, FullDirectoryIndexHash, out var ComputedHash))
            {
                throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
                //UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
                //UE_LOG(LogPakFile, Log, TEXT(" Encrypted: %d"), Info.bEncryptedIndex);
                //UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
                //UE_LOG(LogPakFile, Log, TEXT(" Index Offset: %d"), FullDirectoryIndexOffset);
                //UE_LOG(LogPakFile, Log, TEXT(" Index Size: %d"), FullDirectoryIndexSize);
                //UE_LOG(LogPakFile, Log, TEXT(" Stored Index Hash: %s"), *PathHashIndexHash.ToString());
                //UE_LOG(LogPakFile, Log, TEXT(" Computed Index Hash: %s"), *ComputedHash.ToString());
            }

            BinaryReader PathHashIndexReader = new BinaryReader(new MemoryStream(PathHashIndexData));

            FPakDirectoryEntry[] PathHashIndex = PathHashIndexReader.ReadTArray(() => new FPakDirectoryEntry(PathHashIndexReader));

            dict = new Dictionary <string, FPakEntry>(NumEntries);
            foreach (FPakDirectoryEntry directoryEntry in PathHashIndex)
            {
                foreach (FPathHashIndexEntry hashIndexEntry in directoryEntry.Entries)
                {
                    var path = directoryEntry.Directory + hashIndexEntry.Filename;
                    if (path.StartsWith("/"))
                    {
                        path = path.Substring(1);
                    }
                    if (!CaseSensitive)
                    {
                        path = path.ToLowerInvariant();
                    }
                    // if there is no filter OR the filter passes
                    if (filter == null || filter.CheckFilter(MountPoint + hashIndexEntry.Filename, CaseSensitive))
                    {
                        // Filename is without the MountPoint concatenated to save memory
                        dict[path] = GetEntry(path, hashIndexEntry.Location, EncodedPakEntries);
                    }
                }
            }
        }
Example #2
0
        void ReadIndexUpdated(BinaryReader reader, byte[] aesKey, long totalSize, PakFilter filter)
        {
            MountPoint = reader.ReadFString();
            if (MountPoint.StartsWith("../../.."))
            {
                MountPoint = MountPoint.Substring(8);
            }
            else
            {
                // Weird mount point location...
                MountPoint = "/";
            }
            if (!CaseSensitive)
            {
                MountPoint = MountPoint.ToLowerInvariant();
            }
            var NumEntries   = reader.ReadInt32();
            var PathHashSeed = reader.ReadUInt64();

            bool     bReaderHasPathHashIndex = false;
            long     PathHashIndexOffset     = -1; // INDEX_NONE
            long     PathHashIndexSize       = 0;
            FSHAHash PathHashIndexHash       = default;

            bReaderHasPathHashIndex = reader.ReadInt32() != 0;
            if (bReaderHasPathHashIndex)
            {
                PathHashIndexOffset     = reader.ReadInt64();
                PathHashIndexSize       = reader.ReadInt64();
                PathHashIndexHash       = new FSHAHash(reader);
                bReaderHasPathHashIndex = bReaderHasPathHashIndex && PathHashIndexOffset != -1;
            }

            bool     bReaderHasFullDirectoryIndex = false;
            long     FullDirectoryIndexOffset     = -1; // INDEX_NONE
            long     FullDirectoryIndexSize       = 0;
            FSHAHash FullDirectoryIndexHash       = default;

            bReaderHasFullDirectoryIndex = reader.ReadInt32() != 0;
            if (bReaderHasFullDirectoryIndex)
            {
                FullDirectoryIndexOffset     = reader.ReadInt64();
                FullDirectoryIndexSize       = reader.ReadInt64();
                FullDirectoryIndexHash       = new FSHAHash(reader);
                bReaderHasFullDirectoryIndex = bReaderHasFullDirectoryIndex && FullDirectoryIndexOffset != -1;
            }

            byte[] EncodedPakEntries = reader.ReadTArray(() => reader.ReadByte());
            File.WriteAllBytes("pakentryencoded", EncodedPakEntries);

            int FilesNum = reader.ReadInt32();

            if (FilesNum < 0)
            {
                // Should not be possible for any values in the PrimaryIndex to be invalid, since we verified the index hash
                throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
            }
            FPakEntry[] Files = new FPakEntry[FilesNum]; // from what i can see, there aren't any???
            if (FilesNum > 0)
            {
                for (int FileIndex = 0; FileIndex < FilesNum; ++FileIndex)
                {
                    Files[FileIndex] = new FPakEntry(reader, Info.Version);
                }
            }

            // Decide which SecondaryIndex(es) to load
            bool bWillUseFullDirectoryIndex;
            bool bWillUsePathHashIndex;
            bool bReadFullDirectoryIndex;

            if (bReaderHasPathHashIndex && bReaderHasFullDirectoryIndex)
            {
                bWillUseFullDirectoryIndex = false; // https://github.com/EpicGames/UnrealEngine/blob/79a64829237ae339118bb50b61d84e4599c14e8a/Engine/Source/Runtime/PakFile/Private/IPlatformFilePak.cpp#L5628
                bWillUsePathHashIndex      = !bWillUseFullDirectoryIndex;
                bool bWantToReadFullDirectoryIndex = false;
                bReadFullDirectoryIndex = bReaderHasFullDirectoryIndex && bWantToReadFullDirectoryIndex;
            }
            else if (bReaderHasPathHashIndex)
            {
                bWillUsePathHashIndex      = true;
                bWillUseFullDirectoryIndex = false;
                bReadFullDirectoryIndex    = false;
            }
            else if (bReaderHasFullDirectoryIndex)
            {
                // We don't support creating the PathHash Index at runtime; we want to move to having only the PathHashIndex, so supporting not having it at all is not useful enough to write
                bWillUsePathHashIndex      = false;
                bWillUseFullDirectoryIndex = true;
                bReadFullDirectoryIndex    = true;
            }
            else
            {
                // It should not be possible for PrimaryIndexes to be built without a PathHashIndex AND without a FullDirectoryIndex; CreatePakFile in UnrealPak.exe has a check statement for it.
                throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
            }

            // Load the Secondary Index(es)
            byte[] PathHashIndexData;
            Dictionary <ulong, int> PathHashIndex;
            BinaryReader            PathHashIndexReader = default;
            bool bHasPathHashIndex;

            if (bWillUsePathHashIndex)
            {
                if (PathHashIndexOffset < 0 || totalSize < (PathHashIndexOffset + PathHashIndexSize))
                {
                    // Should not be possible for these values (which came from the PrimaryIndex) to be invalid, since we verified the index hash of the PrimaryIndex
                    throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
                    //UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
                    //UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
                    //UE_LOG(LogPakFile, Log, TEXT(" PathHashIndexOffset : %d"), PathHashIndexOffset);
                    //UE_LOG(LogPakFile, Log, TEXT(" PathHashIndexSize: %d"), PathHashIndexSize);
                }
                Reader.BaseStream.Position = PathHashIndexOffset;
                PathHashIndexData          = Reader.ReadBytes((int)PathHashIndexSize);
                File.WriteAllBytes("indexdata.daa", PathHashIndexData);

                {
                    if (!DecryptAndValidateIndex(Reader, ref PathHashIndexData, aesKey, PathHashIndexHash, out var ComputedHash))
                    {
                        throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
                        //UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
                        //UE_LOG(LogPakFile, Log, TEXT(" Encrypted: %d"), Info.bEncryptedIndex);
                        //UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
                        //UE_LOG(LogPakFile, Log, TEXT(" Index Offset: %d"), FullDirectoryIndexOffset);
                        //UE_LOG(LogPakFile, Log, TEXT(" Index Size: %d"), FullDirectoryIndexSize);
                        //UE_LOG(LogPakFile, Log, TEXT(" Stored Index Hash: %s"), *PathHashIndexHash.ToString());
                        //UE_LOG(LogPakFile, Log, TEXT(" Computed Index Hash: %s"), *ComputedHash.ToString());
                    }
                }

                PathHashIndexReader = new BinaryReader(new MemoryStream(PathHashIndexData));
                PathHashIndex       = ReadPathHashIndex(PathHashIndexReader);
                bHasPathHashIndex   = true;
            }

            var  DirectoryIndex = new Dictionary <string, Dictionary <string, int> >();
            bool bHasFullDirectoryIndex;

            if (!bReadFullDirectoryIndex)
            {
                DirectoryIndex         = ReadDirectoryIndex(PathHashIndexReader);
                bHasFullDirectoryIndex = false;
            }
            if (DirectoryIndex.Count == 0)
            {
                if (totalSize < (FullDirectoryIndexOffset + FullDirectoryIndexSize) ||
                    FullDirectoryIndexOffset < 0)
                {
                    // Should not be possible for these values (which came from the PrimaryIndex) to be invalid, since we verified the index hash of the PrimaryIndex
                    throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
                    //UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
                    //UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
                    //UE_LOG(LogPakFile, Log, TEXT(" FullDirectoryIndexOffset : %d"), FullDirectoryIndexOffset);
                    //UE_LOG(LogPakFile, Log, TEXT(" FullDirectoryIndexSize: %d"), FullDirectoryIndexSize);
                }
                Reader.BaseStream.Position = FullDirectoryIndexOffset;
                byte[] FullDirectoryIndexData = Reader.ReadBytes((int)FullDirectoryIndexSize);

                {
                    if (!DecryptAndValidateIndex(Reader, ref FullDirectoryIndexData, aesKey, FullDirectoryIndexHash, out var ComputedHash))
                    {
                        throw new FileLoadException("Corrupt pak PrimaryIndex detected!");
                        //UE_LOG(LogPakFile, Log, TEXT(" Filename: %s"), *PakFilename);
                        //UE_LOG(LogPakFile, Log, TEXT(" Encrypted: %d"), Info.bEncryptedIndex);
                        //UE_LOG(LogPakFile, Log, TEXT(" Total Size: %d"), Reader->TotalSize());
                        //UE_LOG(LogPakFile, Log, TEXT(" Index Offset: %d"), FullDirectoryIndexOffset);
                        //UE_LOG(LogPakFile, Log, TEXT(" Index Size: %d"), FullDirectoryIndexSize);
                        //UE_LOG(LogPakFile, Log, TEXT(" Stored Index Hash: %s"), *FullDirectoryIndexHash.ToString());
                        //UE_LOG(LogPakFile, Log, TEXT(" Computed Index Hash: %s"), *ComputedHash.ToString());
                    }
                }

                var SecondaryIndexReader = new BinaryReader(new MemoryStream(FullDirectoryIndexData));
                DirectoryIndex         = ReadDirectoryIndex(SecondaryIndexReader);
                bHasFullDirectoryIndex = true;
            }

            Entries = new Dictionary <string, FPakEntry>(NumEntries);
            foreach (var(dirname, dir) in DirectoryIndex)
            {
                foreach (var(filename, pakLocation) in dir)
                {
                    var path = dirname + filename;
                    if (!CaseSensitive)
                    {
                        path = path.ToLowerInvariant();
                    }
                    // if there is no filter OR the filter passes
                    if (filter == null || filter.CheckFilter(MountPoint + filename, CaseSensitive))
                    {
                        // Filename is without the MountPoint concatenated to save memory
                        Entries[path] = GetEntry(pakLocation, EncodedPakEntries);
                    }
                }
            }
        }
Example #3
0
        void ReadIndexInternal(byte[] key, PakFilter filter, out Exception exc)
        {
            if (Initialized)
            {
                exc = new InvalidOperationException("Index is already initialized");
                return;
            }

            if (Info.bEncryptedIndex && key == null)
            {
                exc = new ArgumentException("Index is encrypted but no key was provided", nameof(key));
                return;
            }

            Stream.Position = Info.IndexOffset;

            BinaryReader IndexReader;

            if (Info.bEncryptedIndex)
            {
                IndexReader = new BinaryReader(new MemoryStream(AESDecryptor.DecryptAES(Reader.ReadBytes((int)Info.IndexSize), key)));
                int stringLen = IndexReader.ReadInt32();
                if (stringLen > 512 || stringLen < -512)
                {
                    exc = new ArgumentException("The provided key is invalid", nameof(key));
                    return;
                }
                if (stringLen < 0)
                {
                    IndexReader.BaseStream.Position += (stringLen - 1) * 2;
                    if (IndexReader.ReadUInt16() != 0)
                    {
                        exc = new ArgumentException("The provided key is invalid", nameof(key));
                        return;
                    }
                }
                else
                {
                    IndexReader.BaseStream.Position += stringLen - 1;
                    if (IndexReader.ReadByte() != 0)
                    {
                        exc = new ArgumentException("The provided key is invalid", nameof(key));
                        return;
                    }
                }
                IndexReader.BaseStream.Position = 0;
            }
            else
            {
                IndexReader = Reader;
            }

            Dictionary <string, FPakEntry> tempFiles;

            if (Info.Version >= EPakVersion.PATH_HASH_INDEX)
            {
                ReadIndexUpdated(IndexReader, key, out tempFiles, filter);
            }
            else
            {
                // https://github.com/EpicGames/UnrealEngine/blob/bf95c2cbc703123e08ab54e3ceccdd47e48d224a/Engine/Source/Runtime/PakFile/Private/IPlatformFilePak.cpp#L4509
                MountPoint = IndexReader.ReadFString();
                if (MountPoint.StartsWith("../../.."))
                {
                    MountPoint = MountPoint.Substring(8);
                }
                else
                {
                    // Weird mount point location...
                    MountPoint = "/";
                }
                if (!CaseSensitive)
                {
                    MountPoint = MountPoint.ToLowerInvariant();
                }

                var NumEntries = IndexReader.ReadInt32();
                tempFiles = new Dictionary <string, FPakEntry>(NumEntries);
                for (int i = 0; i < NumEntries; i++)
                {
                    var entry = new FPakEntry(IndexReader, Info.Version, CaseSensitive, FileName);
                    // if there is no filter OR the filter passes
                    if (filter == null || filter.CheckFilter(MountPoint + entry.Name, CaseSensitive))
                    {
                        // Filename is without the MountPoint concatenated to save memory
                        tempFiles[entry.Name] = entry;
                    }
                }
            }

            Paks.Merge(tempFiles, out var files, MountPoint);
            Entries = files;

            DebugHelper.WriteLine("{0} {1} {2} {3}", "[FModel]", "[PakFileReader]", "[ReadIndexInternal]", $"{FileName} contains {Entries.Count} files, mount point: \"{this.MountPoint}\", version: {(int)this.Info.Version}");

            if (Info.bEncryptedIndex)
            {
                // underlying stream is a MemoryStream of the decrypted index, might improve performance with a crypto stream of some sort
                IndexReader.Dispose();
            }

            Reader.Dispose();
            Initialized = true;
            exc         = null;
        }
Example #4
0
        void ReadIndexInternal(byte[] key, PakFilter filter, out Exception exc)
        {
            if (Initialized)
            {
                exc = new InvalidOperationException("Index is already initialized");
                return;
            }

            if (Info.bEncryptedIndex && key == null)
            {
                exc = new ArgumentException("Index is encrypted but no key was provided", nameof(key));
                return;
            }

            Stream.Position = Info.IndexOffset;

            BinaryReader IndexReader;

            if (Info.bEncryptedIndex)
            {
                IndexReader = new BinaryReader(new MemoryStream(AESDecryptor.DecryptAES(Reader.ReadBytes((int)Info.IndexSize), key)));
                int stringLen = IndexReader.ReadInt32();
                if (stringLen > 512 || stringLen < -512)
                {
                    exc = new ArgumentException("The provided key is invalid", nameof(key));
                    return;
                }
                if (stringLen < 0)
                {
                    IndexReader.BaseStream.Position += (stringLen - 1) * 2;
                    if (IndexReader.ReadUInt16() != 0)
                    {
                        exc = new ArgumentException("The provided key is invalid", nameof(key));
                        return;
                    }
                }
                else
                {
                    IndexReader.BaseStream.Position += stringLen - 1;
                    if (IndexReader.ReadByte() != 0)
                    {
                        exc = new ArgumentException("The provided key is invalid", nameof(key));
                        return;
                    }
                }
                IndexReader.BaseStream.Position = 0;
            }
            else
            {
                IndexReader = Reader;
            }

            Dictionary <string, FPakEntry> tempFiles;

            if (Info.Version >= EPakVersion.PATH_HASH_INDEX)
            {
                ReadIndexUpdated(IndexReader, key, out tempFiles, filter);
            }
            else
            {
                // https://github.com/EpicGames/UnrealEngine/blob/bf95c2cbc703123e08ab54e3ceccdd47e48d224a/Engine/Source/Runtime/PakFile/Private/IPlatformFilePak.cpp#L4509
                MountPoint = IndexReader.ReadFString() ?? "";
                if (MountPoint.StartsWith("../../.."))
                {
                    MountPoint = MountPoint[8..];