private static byte[] ParseBLTEfile(byte[] content) { MemoryStream result = new MemoryStream(); using (BinaryReader bin = new BinaryReader(new MemoryStream(content))) { if (bin.ReadUInt32() != 0x45544c42) { throw new Exception("Not a BLTE file"); } var blteSize = bin.ReadUInt32(true); BLTEChunkInfo[] chunkInfos; if (blteSize == 0) { chunkInfos = new BLTEChunkInfo[1]; chunkInfos[0].isFullChunk = false; chunkInfos[0].inFileSize = Convert.ToInt32(bin.BaseStream.Length - bin.BaseStream.Position); chunkInfos[0].actualSize = Convert.ToInt32(bin.BaseStream.Length - bin.BaseStream.Position); chunkInfos[0].checkSum = new byte[16];; } else { var bytes = bin.ReadBytes(4); var chunkCount = bytes[1] << 16 | bytes[2] << 8 | bytes[3] << 0; //var unk = bin.ReadByte(); ////Code by TOM_RUS //byte v1 = bin.ReadByte(); //byte v2 = bin.ReadByte(); //byte v3 = bin.ReadByte(); //var chunkCount = v1 << 16 | v2 << 8 | v3 << 0; // 3-byte ////Retrieved from https://github.com/WoW-Tools/CASCExplorer/blob/cli/CascLib/BLTEHandler.cs#L76 var supposedHeaderSize = 24 * chunkCount + 12; if (supposedHeaderSize != blteSize) { throw new Exception("Invalid header size!"); } if (supposedHeaderSize > bin.BaseStream.Length) { throw new Exception("Not enough data"); } chunkInfos = new BLTEChunkInfo[chunkCount]; for (int i = 0; i < chunkCount; i++) { chunkInfos[i].isFullChunk = true; chunkInfos[i].inFileSize = bin.ReadInt32(true); chunkInfos[i].actualSize = bin.ReadInt32(true); chunkInfos[i].checkSum = new byte[16]; chunkInfos[i].checkSum = bin.ReadBytes(16); } } foreach (var chunk in chunkInfos) { MemoryStream chunkResult = new MemoryStream(); if (chunk.inFileSize > bin.BaseStream.Length) { throw new Exception("Trying to read more than is available!"); } var chunkBuffer = bin.ReadBytes(chunk.inFileSize); var hasher = MD5.Create(); var md5sum = hasher.ComputeHash(chunkBuffer); if (chunk.isFullChunk && BitConverter.ToString(md5sum) != BitConverter.ToString(chunk.checkSum)) { throw new Exception("MD5 checksum mismatch on BLTE chunk! Sum is " + BitConverter.ToString(md5sum).Replace("-", "") + " but is supposed to be " + BitConverter.ToString(chunk.checkSum).Replace("-", "")); } using (BinaryReader chunkreader = new BinaryReader(new MemoryStream(chunkBuffer))) { var mode = chunkreader.ReadChar(); switch (mode) { case 'N': // none chunkResult.Write(chunkreader.ReadBytes(chunk.actualSize), 0, chunk.actualSize); //read actual size because we already read the N from chunkreader break; case 'Z': // zlib, todo using (MemoryStream stream = new MemoryStream(chunkreader.ReadBytes(chunk.inFileSize - 1), 2, chunk.inFileSize - 3)) { var ds = new DeflateStream(stream, CompressionMode.Decompress); ds.CopyTo(chunkResult); } break; case 'F': // frame case 'E': // encrypted Console.WriteLine("Encrypted file!"); break; default: throw new Exception("Unsupported mode!"); } if (mode == 'N' || mode == 'Z') { var chunkres = chunkResult.ToArray(); if (chunk.isFullChunk && chunkres.Length != chunk.actualSize) { throw new Exception("Decoded result is wrong size!"); } result.Write(chunkres, 0, chunkres.Length); } else { Console.WriteLine("Unsupported file (Encrypted?)"); } } } foreach (var chunk in chunkInfos) { if (chunk.inFileSize > bin.BaseStream.Length) { throw new Exception("Trying to read more than is available!"); } else { bin.BaseStream.Position += chunk.inFileSize; } } } return(result.ToArray()); }
static unsafe void Main(string[] args) { if (args.Count() < 1) { throw new ArgumentOutOfRangeException("Program requires at least one argument: directory that it will need to verify all contents of and optionally a location of local indexes/archives"); } if (args.Count() > 1) { foreach (var file in Directory.GetFiles(args[1], "*.idx", SearchOption.AllDirectories)) { using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var br = new BinaryReader(stream)) { int h2Len = br.ReadInt32(); int h2Check = br.ReadInt32(); byte[] h2 = br.ReadBytes(h2Len); long padPos = (8 + h2Len + 0x0F) & 0xFFFFFFF0; stream.Position = padPos; int dataLen = br.ReadInt32(); int dataCheck = br.ReadInt32(); int numBlocks = dataLen / 18; for (int i = 0; i < numBlocks; i++) { var info = new IndexEntry(); byte[] keyBytes = br.ReadBytes(9); Array.Resize(ref keyBytes, 16); MD5Hash key; fixed(byte *ptr = keyBytes) key = *(MD5Hash *)ptr; //Console.WriteLine(key.ToHexString()); byte indexHigh = br.ReadByte(); int indexLow = br.ReadInt32BE(); info.Index = (indexHigh << 2 | (byte)((indexLow & 0xC0000000) >> 30)); info.Offset = (indexLow & 0x3FFFFFFF); info.Size = br.ReadInt32(); if (!LocalIndexData.ContainsKey(key)) // use first key { LocalIndexData.Add(key, info); } } padPos = (dataLen + 0x0FFF) & 0xFFFFF000; stream.Position = padPos; stream.Position += numBlocks * 18; } } } Console.WriteLine("Reading all archives on disk.."); var archiveList = new List <MD5Hash>(); foreach (var file in Directory.GetFiles(Path.Combine(args[0], "tpr", "wow", "data"), "*.index", SearchOption.AllDirectories)) { var indexName = Path.GetFileNameWithoutExtension(file); using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var bin = new BinaryReader(stream)) { bin.BaseStream.Position = bin.BaseStream.Length - 16; // 0 = loose file index, 4 = archive index, 6 = group archive index if (bin.ReadChar() == 4) { if (!File.Exists(Path.Combine(args[0], "tpr", "wow", "patch", "" + indexName[0] + indexName[1], "" + indexName[2] + indexName[3], indexName + ".index"))) { if (File.Exists(Path.Combine(args[0], "tpr", "wow", "data", "" + indexName[0] + indexName[1], "" + indexName[2] + indexName[3], indexName))) { archiveList.Add(Path.GetFileNameWithoutExtension(file).ToByteArray().ToMD5()); } } } } } //18125-18179 missing var missingArchiveList = new List <MD5Hash> { "5BEBD82DD19DEFD0A0BB2B97C178A59F".ToByteArray().ToMD5(), "5E430F0C8A3F77B2E6F62F4BA74E1147".ToByteArray().ToMD5(), "6F75F753E8CF156FD5B7381F9A63C210".ToByteArray().ToMD5(), "9A8618C3393299DA9BDAD3649A0B7E5D".ToByteArray().ToMD5(), "9F02F9518B4844B85D957A6371C98656".ToByteArray().ToMD5(), "F9B96A5557541391ED3770B85B06D5A8".ToByteArray().ToMD5() }; Console.WriteLine("Checking all archives for files for missing archives.."); Parallel.ForEach(archiveList.ToArray(), (archive, state, i) => { var indexName = archive.ToHexString().ToLower(); var indexContent = File.ReadAllBytes(Path.Combine(args[0], "tpr", "wow", "data", "" + indexName[0] + indexName[1], "" + indexName[2] + indexName[3], indexName + ".index")); var foundCount = 0; var totalCount = 0; using (var indexBin = new BinaryReader(new MemoryStream(indexContent))) { var indexEntries = indexContent.Length / 4096; FileStream stream; var archiveBin = new BinaryReader(new MemoryStream()); if (!missingArchiveList.Contains(archive)) { stream = new FileStream(Path.Combine(args[0], "tpr", "wow", "data", "" + indexName[0] + indexName[1], "" + indexName[2] + indexName[3], indexName), FileMode.Open, FileAccess.Read, FileShare.Read); archiveBin = new BinaryReader(stream); } for (var b = 0; b < indexEntries; b++) { for (var bi = 0; bi < 170; bi++) { var headerHash = indexBin.Read <MD5Hash>(); var size = indexBin.ReadUInt32(true); var offset = indexBin.ReadUInt32(true); if (headerHash.ToHexString() == "00000000000000000000000000000000") { continue; } if (!missingArchiveList.Contains(archive)) { if (offset > archiveBin.BaseStream.Length) { Console.WriteLine("[" + indexName + "] Unable to read hash " + headerHash.ToHexString() + ", offset " + offset + " is after end of stream " + archiveBin.BaseStream.Length + "!"); } else if ((offset + size) > archiveBin.BaseStream.Length) { Console.WriteLine("[" + indexName + "] Unable to read hash " + headerHash.ToHexString() + ", offset " + offset + "+" + size + " goes on beyond end of stream " + archiveBin.BaseStream.Length + "!"); } else { archiveBin.BaseStream.Position = offset; if (archiveBin.ReadUInt64() == 0) { Console.WriteLine("[" + indexName + "] Unable to read hash " + headerHash.ToHexString() + ", offset " + offset + "+" + size + ", it starts with all 0s!"); } } } else { var indexEntry = GetIndexInfo(headerHash); if (indexEntry != null) { foundCount++; Console.WriteLine("[" + indexName + "] Unable to read hash " + headerHash.ToHexString() + ", offset " + offset + " size " + size + " since archive does not exist but file exists in local archives in archive " + indexEntry.Index + " at offset " + indexEntry.Offset + " with size " + indexEntry.Size + "!"); } else { Console.WriteLine("[" + indexName + "] Unable to read hash " + headerHash.ToHexString() + ", offset " + offset + " archive does not exist!"); } totalCount++; } } indexBin.ReadBytes(16); } if (missingArchiveList.Contains(archive)) { Console.WriteLine("Found " + foundCount + " of " + totalCount + " files for archive " + indexName + " in local archives!"); } } }); Environment.Exit(0); //GetIndexes(Path.Combine(CDN.cacheDir, "tpr", "wow"), archiveList.ToArray()); Console.WriteLine("Listing directory.."); var files = new string[0]; // Hackfix to read from txt file instead if (args[0].EndsWith(".txt")) { files = File.ReadAllLines(args[0]); } else { files = Directory.GetFiles(Path.Combine(args[0], "tpr", "wow"), "*", SearchOption.AllDirectories); } foreach (var file in files) { Console.WriteLine(file); if (file.Contains("config")) { using (var md5 = MD5.Create()) { using (var stream = File.OpenRead(file)) { var md5sum = BitConverter.ToString(md5.ComputeHash(stream)).ToLower().Replace("-", ""); if (md5sum != Path.GetFileName(file)) { throw new Exception(" Invalid config file! MD5 of " + Path.GetFileName(file) + " is " + md5sum); } } } continue; } if (file.EndsWith(".index")) { continue; } using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var bin = new BinaryReader(stream)) { try { var header = bin.ReadUInt32(); if (header != 0x45544C42) { Console.WriteLine(" Invalid BLTE file! " + header); if (header == 0x4453425A) { Console.WriteLine(" File is a patch archive!"); } else if (header == 268583248) { Console.WriteLine(" File is a patch index!"); } else if (header == 893668643 || header == 1279870499) { Console.WriteLine(" File is Overwatch root file!"); } else { Console.WriteLine("Unknown file encountered!"); Console.ReadLine(); } } else { var blteSize = bin.ReadUInt32(true); BLTEChunkInfo[] chunkInfos; if (blteSize == 0) { Console.WriteLine(file + " is a single chunk file!"); chunkInfos = new BLTEChunkInfo[1]; chunkInfos[0].isFullChunk = false; chunkInfos[0].inFileSize = Convert.ToInt32(bin.BaseStream.Length - bin.BaseStream.Position); chunkInfos[0].actualSize = Convert.ToInt32(bin.BaseStream.Length - bin.BaseStream.Position); chunkInfos[0].checkSum = new byte[16]; } else { var bytes = bin.ReadBytes(4); //Code by TOM_RUS //Retrieved from https://github.com/WoW-Tools/CASCExplorer/blob/cli/CascLib/BLTEHandler.cs#L76 var chunkCount = bytes[1] << 16 | bytes[2] << 8 | bytes[3] << 0; var supposedHeaderSize = 24 * chunkCount + 12; if (supposedHeaderSize != blteSize) { File.AppendAllText("bad.txt", file + " (Invalid header size) " + Environment.NewLine); } if (supposedHeaderSize > bin.BaseStream.Length) { File.AppendAllText("bad.txt", file + " (Not enough data) " + Environment.NewLine); } chunkInfos = new BLTEChunkInfo[chunkCount]; for (var i = 0; i < chunkCount; i++) { chunkInfos[i].isFullChunk = true; chunkInfos[i].inFileSize = bin.ReadInt32(true); chunkInfos[i].actualSize = bin.ReadInt32(true); chunkInfos[i].checkSum = new byte[16]; chunkInfos[i].checkSum = bin.ReadBytes(16); } foreach (var chunk in chunkInfos) { using (var chunkResult = new MemoryStream()) { if (chunk.inFileSize > bin.BaseStream.Length) { File.AppendAllText("bad.txt", file + " (Not enough data remaining in stream) " + Environment.NewLine); continue; } var chunkBuffer = bin.ReadBytes(chunk.inFileSize); using (var hasher = MD5.Create()) { var md5sum = hasher.ComputeHash(chunkBuffer); if (chunk.isFullChunk && BitConverter.ToString(md5sum) != BitConverter.ToString(chunk.checkSum)) { File.AppendAllText("bad.txt", file + " (chunk md5sum mismatch) " + Environment.NewLine); chunkBuffer = null; continue; } } using (var chunkms = new MemoryStream(chunkBuffer)) using (var chunkreader = new BinaryReader(chunkms)) { var mode = chunkreader.ReadChar(); switch (mode) { case 'N': // none chunkResult.Write(chunkreader.ReadBytes(chunk.actualSize), 0, chunk.actualSize); //read actual size because we already read the N from chunkreader break; case 'Z': // zlib, todo using (var mstream = new MemoryStream(chunkreader.ReadBytes(chunk.inFileSize - 1), 2, chunk.inFileSize - 3)) using (var ds = new DeflateStream(mstream, CompressionMode.Decompress)) { ds.CopyTo(chunkResult); } break; case 'E': // encrypted //Console.WriteLine("Encrypted file!"); break; case 'F': // frame default: throw new Exception("Unsupported mode!"); } // Don't check integrity for unsupported chunks if (mode == 'N' || mode == 'Z') { var chunkres = chunkResult.ToArray(); if (chunk.isFullChunk && chunkres.Length != chunk.actualSize) { File.AppendAllText("bad.txt", file + " (bad chunk result size) " + Environment.NewLine); } chunkres = null; } chunkBuffer = null; } } } chunkInfos = null; } } } catch (EndOfStreamException e) { File.AppendAllText("bad.txt", file + Environment.NewLine); } } } }
static void Main(string[] args) { if (args.Count() != 1) { throw new ArgumentOutOfRangeException("Program requires one argument: directory that it will need to verify all contents of"); } Console.WriteLine("Listing directory.."); string[] files = new string[0]; // Hackfix to read from txt file instead if (args[0].EndsWith(".txt")) { files = File.ReadAllLines(args[0]); } else { files = Directory.GetFiles(args[0], "*", SearchOption.AllDirectories); } foreach (string file in files) { Console.WriteLine(file); if (file.Contains("config")) { using (var md5 = MD5.Create()) { using (var stream = File.OpenRead(file)) { var md5sum = BitConverter.ToString(md5.ComputeHash(stream)).ToLower().Replace("-", ""); if (md5sum != Path.GetFileName(file)) { throw new Exception(" Invalid config file! MD5 of " + Path.GetFileName(file) + " is " + md5sum); } } } continue; } if (file.EndsWith(".index")) { //Console.WriteLine(" Not checking indexes"); continue; } using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var bin = new BinaryReader(stream)) { try { var header = bin.ReadUInt32(); if (header != 0x45544C42) { Console.WriteLine(" Invalid BLTE file! " + header); if (header == 0x4453425A) { Console.WriteLine(" File is a patch archive!"); } else if (header == 268583248) { Console.WriteLine(" File is a patch index!"); } else { Console.WriteLine("Unknown file encountered!"); Console.ReadLine(); } } else { var blteSize = bin.ReadUInt32(true); BLTEChunkInfo[] chunkInfos; if (blteSize == 0) { chunkInfos = new BLTEChunkInfo[1]; chunkInfos[0].isFullChunk = false; chunkInfos[0].inFileSize = Convert.ToInt32(bin.BaseStream.Length - bin.BaseStream.Position); chunkInfos[0].actualSize = Convert.ToInt32(bin.BaseStream.Length - bin.BaseStream.Position); chunkInfos[0].checkSum = new byte[16]; } else { var bytes = bin.ReadBytes(4); //Code by TOM_RUS //Retrieved from https://github.com/WoW-Tools/CASCExplorer/blob/cli/CascLib/BLTEHandler.cs#L76 var chunkCount = bytes[1] << 16 | bytes[2] << 8 | bytes[3] << 0; var supposedHeaderSize = 24 * chunkCount + 12; if (supposedHeaderSize != blteSize) { File.AppendAllText("bad.txt", file + " (Invalid header size) " + Environment.NewLine); } if (supposedHeaderSize > bin.BaseStream.Length) { File.AppendAllText("bad.txt", file + " (Not enough data) " + Environment.NewLine); } chunkInfos = new BLTEChunkInfo[chunkCount]; for (int i = 0; i < chunkCount; i++) { chunkInfos[i].isFullChunk = true; chunkInfos[i].inFileSize = bin.ReadInt32(true); chunkInfos[i].actualSize = bin.ReadInt32(true); chunkInfos[i].checkSum = new byte[16]; chunkInfos[i].checkSum = bin.ReadBytes(16); } foreach (var chunk in chunkInfos) { using (MemoryStream chunkResult = new MemoryStream()) { if (chunk.inFileSize > bin.BaseStream.Length) { File.AppendAllText("bad.txt", file + " (Not enough data remaining in stream) " + Environment.NewLine); continue; } var chunkBuffer = bin.ReadBytes(chunk.inFileSize); using (var hasher = MD5.Create()) { var md5sum = hasher.ComputeHash(chunkBuffer); if (chunk.isFullChunk && BitConverter.ToString(md5sum) != BitConverter.ToString(chunk.checkSum)) { File.AppendAllText("bad.txt", file + " (chunk md5sum mismatch) " + Environment.NewLine); chunkBuffer = null; continue; } } using (MemoryStream chunkms = new MemoryStream(chunkBuffer)) using (BinaryReader chunkreader = new BinaryReader(chunkms)) { var mode = chunkreader.ReadChar(); switch (mode) { case 'N': // none chunkResult.Write(chunkreader.ReadBytes(chunk.actualSize), 0, chunk.actualSize); //read actual size because we already read the N from chunkreader break; case 'Z': // zlib, todo using (MemoryStream mstream = new MemoryStream(chunkreader.ReadBytes(chunk.inFileSize - 1), 2, chunk.inFileSize - 3)) using (DeflateStream ds = new DeflateStream(mstream, CompressionMode.Decompress)) { ds.CopyTo(chunkResult); } break; case 'E': // encrypted //Console.WriteLine("Encrypted file!"); break; case 'F': // frame default: throw new Exception("Unsupported mode!"); } // Don't check integrity for unsupported chunks if (mode == 'N' || mode == 'Z') { var chunkres = chunkResult.ToArray(); if (chunk.isFullChunk && chunkres.Length != chunk.actualSize) { File.AppendAllText("bad.txt", file + " (bad chunk result size) " + Environment.NewLine); } chunkres = null; } chunkBuffer = null; } } } chunkInfos = null; } } } catch (EndOfStreamException e) { File.AppendAllText("bad.txt", file + Environment.NewLine); } } } }