public static async Task <CDNConfigFile> GetCDNConfig(string hash)
{
var cdnConfig = new CDNConfigFile();
string content = System.Text.Encoding.UTF8.GetString(await CDNCache.Get("config", hash));
var cdnConfigLines = content.Split(new string[] { "\n" }, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < cdnConfigLines.Count(); i++)
{
if (cdnConfigLines[i].StartsWith("#") || cdnConfigLines[i].Length == 0)
{
continue;
}
var cols = cdnConfigLines[i].Split(new string[] { " = " }, StringSplitOptions.RemoveEmptyEntries);
switch (cols[0])
{
case "archives":
var archives = cols[1].Split(' ');
cdnConfig.archives = new MD5Hash[archives.Length];
for (var j = 0; j < archives.Length; j++)
{
cdnConfig.archives[j] = archives[j].ToByteArray().ToMD5();
}
break;
default:
break;
}
}
return(cdnConfig);
}
public async static Task <byte[]> RetrieveFileBytes(MD5Hash target)
{
var targetString = target.ToHexString().ToLower();
var unarchivedName = Path.Combine(CDNCache.cacheDir, "tpr/wow", "data", targetString[0] + "" + targetString[1], targetString[2] + "" + targetString[3], targetString);
if (File.Exists(unarchivedName))
{
return(BLTE.Parse(await File.ReadAllBytesAsync(unarchivedName)));
}
if (!indexDictionary.TryGetValue(target, out IndexEntry entry))
{
throw new Exception("Unable to find file in archives. File is not available!?");
}
var index = indexNames[(int)entry.indexID].ToHexString().ToLower();
var archiveName = Path.Combine(CDNCache.cacheDir, "tpr/wow", "data", index[0] + "" + index[1], index[2] + "" + index[3], index);
if (!File.Exists(archiveName))
{
Logger.WriteLine("Unable to find archive " + index + " on disk, attempting to stream from CDN instead");
try
{
return(BLTE.Parse(await CDNCache.Get("data", index, true, false, entry.size, entry.offset)));
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
else
{
using (var stream = new FileStream(archiveName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
stream.Seek(entry.offset, SeekOrigin.Begin);
try
{
if (entry.offset > stream.Length || entry.offset + entry.size > stream.Length)
{
throw new Exception("File is beyond archive length, incomplete archive!");
}
var archiveBytes = new byte[entry.size];
await stream.ReadAsync(archiveBytes, 0, (int)entry.size);
return(BLTE.Parse(archiveBytes));
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}
return(new byte[0]);
}
public static async Task <InstallFile> GetInstall(string hash, bool parseIt = false)
{
var install = new InstallFile();
byte[] content = await CDNCache.Get("data", hash);
if (!parseIt)
{
return(install);
}
using (BinaryReader bin = new BinaryReader(new MemoryStream(BLTE.Parse(content))))
{
if (Encoding.UTF8.GetString(bin.ReadBytes(2)) != "IN")
{
throw new Exception("Error while parsing install file. Did BLTE header size change?");
}
bin.ReadByte();
install.hashSize = bin.ReadByte();
if (install.hashSize != 16)
{
throw new Exception("Unsupported install hash size!");
}
install.numTags = bin.ReadUInt16(true);
install.numEntries = bin.ReadUInt32(true);
int bytesPerTag = ((int)install.numEntries + 7) / 8;
install.tags = new InstallTagEntry[install.numTags];
for (var i = 0; i < install.numTags; i++)
{
install.tags[i].name = bin.ReadCString();
install.tags[i].type = bin.ReadUInt16(true);
var filebits = bin.ReadBytes(bytesPerTag);
for (int j = 0; j < bytesPerTag; j++)
{
filebits[j] = (byte)((filebits[j] * 0x0202020202 & 0x010884422010) % 1023);
}
install.tags[i].files = new BitArray(filebits);
}
install.entries = new InstallFileEntry[install.numEntries];
for (var i = 0; i < install.numEntries; i++)
{
install.entries[i].name = bin.ReadCString();
install.entries[i].contentHash = BitConverter.ToString(bin.ReadBytes(install.hashSize)).Replace("-", "").ToLower();
install.entries[i].size = bin.ReadUInt32(true);
install.entries[i].tags = new List <string>();
for (var j = 0; j < install.numTags; j++)
{
if (install.tags[j].files[i] == true)
{
install.entries[i].tags.Add(install.tags[j].type + "=" + install.tags[j].name);
}
}
}
}
return(install);
}
public static void GetIndexes(string url, MD5Hash[] archives)
{
Parallel.ForEach(archives, async(archive, state, i) =>
{
uint indexID;
string indexName = archive.ToHexString().ToLower();
try
{
indexCacheLock.EnterUpgradeableReadLock();
if (!CASC.indexNames.Contains(archives[i], new MD5HashComparer()))
{
try
{
indexCacheLock.EnterWriteLock();
CASC.indexNames.Add(archive);
indexID = (uint)CASC.indexNames.Count - 1;
}
finally
{
indexCacheLock.ExitWriteLock();
}
}
else
{
return;
}
}
finally
{
indexCacheLock.ExitUpgradeableReadLock();
}
long archiveLength = 0;
if (File.Exists(Path.Combine(url, "data", "" + indexName[0] + indexName[1], "" + indexName[2] + indexName[3], indexName)))
{
Console.WriteLine("WARNING! Archive " + indexName + " not found, skipping bound checks!");
archiveLength = new FileInfo(Path.Combine(url, "data", "" + indexName[0] + indexName[1], "" + indexName[2] + indexName[3], indexName)).Length;
}
var indexContent = await CDNCache.Get("data", indexName + ".index");
using BinaryReader bin = new BinaryReader(new MemoryStream(indexContent));
bin.BaseStream.Position = bin.BaseStream.Length - 12;
var entryCount = bin.ReadUInt32();
bin.BaseStream.Position = 0;
int indexEntries = indexContent.Length / 4096;
var entriesRead = 0;
for (var b = 0; b < indexEntries; b++)
{
for (var bi = 0; bi < 170; bi++)
{
var headerHash = bin.Read <MD5Hash>();
var entry = new IndexEntry()
{
indexID = indexID,
size = bin.ReadUInt32(true),
offset = bin.ReadUInt32(true)
};
entriesRead++;
if (archiveLength > 0 && (entry.offset + entry.size) > archiveLength)
{
//Console.WriteLine("Read index entry at " + entry.offset + " of size " + entry.size + " that goes beyond size of archive " + indexName + " " + archiveLength + ", skipping..");
}
else
{
indexCacheLock.EnterUpgradeableReadLock();
try
{
if (!CASC.indexDictionary.ContainsKey(headerHash))
{
indexCacheLock.EnterWriteLock();
try
{
CASC.indexDictionary.Add(headerHash, entry);
}
finally
{
indexCacheLock.ExitWriteLock();
}
}
}
finally
{
indexCacheLock.ExitUpgradeableReadLock();
}
}
}
if (entriesRead == entryCount)
{
return;
}
// 16 bytes padding that rounds the chunk to 4096 bytes (index entry is 24 bytes, 24 * 170 = 4080 bytes so 16 bytes remain)
bin.ReadBytes(16);
}
});
}
public static async Task <RootFile> GetRoot(string hash, bool parseIt = false)
{
var root = new RootFile
{
entriesLookup = new MultiDictionary <ulong, RootEntry>(),
entriesFDID = new MultiDictionary <uint, RootEntry>(),
};
byte[] content = await CDNCache.Get("data", hash);
if (!parseIt)
{
return(root);
}
var namedCount = 0;
var unnamedCount = 0;
var newRoot = false;
using (MemoryStream ms = new MemoryStream(BLTE.Parse(content)))
using (BinaryReader bin = new BinaryReader(ms))
{
var header = bin.ReadUInt32();
if (header == 1296454484)
{
uint totalFiles = bin.ReadUInt32();
uint namedFiles = bin.ReadUInt32();
newRoot = true;
}
else
{
bin.BaseStream.Position = 0;
}
while (bin.BaseStream.Position < bin.BaseStream.Length)
{
var count = bin.ReadUInt32();
var contentFlags = (ContentFlags)bin.ReadUInt32();
var localeFlags = (LocaleFlags)bin.ReadUInt32();
var entries = new RootEntry[count];
var filedataIds = new int[count];
var fileDataIndex = 0;
for (var i = 0; i < count; ++i)
{
entries[i].localeFlags = localeFlags;
entries[i].contentFlags = contentFlags;
filedataIds[i] = fileDataIndex + bin.ReadInt32();
entries[i].fileDataID = (uint)filedataIds[i];
fileDataIndex = filedataIds[i] + 1;
}
if (!newRoot)
{
for (var i = 0; i < count; ++i)
{
entries[i].md5 = bin.Read <MD5Hash>();
entries[i].lookup = bin.ReadUInt64();
root.entriesLookup.Add(entries[i].lookup, entries[i]);
root.entriesFDID.Add(entries[i].fileDataID, entries[i]);
}
}
else
{
for (var i = 0; i < count; ++i)
{
entries[i].md5 = bin.Read <MD5Hash>();
}
for (var i = 0; i < count; ++i)
{
if (contentFlags.HasFlag(ContentFlags.NoNames))
{
entries[i].lookup = 0;
unnamedCount++;
}
else
{
entries[i].lookup = bin.ReadUInt64();
namedCount++;
root.entriesLookup.Add(entries[i].lookup, entries[i]);
}
root.entriesFDID.Add(entries[i].fileDataID, entries[i]);
}
}
}
}
return(root);
}
public static async Task <EncodingFile> GetEncoding(string hash, int encodingSize = 0, bool parseTableB = false, bool checkStuff = false)
{
var encoding = new EncodingFile();
hash = hash.ToLower();
byte[] content = await CDNCache.Get("data", hash);
if (encodingSize != 0 && encodingSize != content.Length)
{
// Re-download file, not expected size.
content = await CDNCache.Get("data", hash, true, true);
if (encodingSize != content.Length && encodingSize != 0)
{
throw new Exception("File corrupt/not fully downloaded! Remove " + "data / " + hash[0] + hash[1] + " / " + hash[2] + hash[3] + " / " + hash + " from cache.");
}
}
using (BinaryReader bin = new BinaryReader(new MemoryStream(BLTE.Parse(content))))
{
if (Encoding.UTF8.GetString(bin.ReadBytes(2)) != "EN")
{
throw new Exception("Error while parsing encoding file. Did BLTE header size change?");
}
encoding.version = bin.ReadByte();
encoding.cKeyLength = bin.ReadByte();
encoding.eKeyLength = bin.ReadByte();
encoding.cKeyPageSize = bin.ReadUInt16(true);
encoding.eKeyPageSize = bin.ReadUInt16(true);
encoding.cKeyPageCount = bin.ReadUInt32(true);
encoding.eKeyPageCount = bin.ReadUInt32(true);
encoding.stringBlockSize = bin.ReadUInt40(true);
var headerLength = bin.BaseStream.Position;
if (parseTableB)
{
var stringBlockEntries = new List <string>();
while ((bin.BaseStream.Position - headerLength) != (long)encoding.stringBlockSize)
{
stringBlockEntries.Add(bin.ReadCString());
}
encoding.stringBlockEntries = stringBlockEntries.ToArray();
}
else
{
bin.BaseStream.Position += (long)encoding.stringBlockSize;
}
/* Table A */
if (checkStuff)
{
encoding.aHeaders = new EncodingHeaderEntry[encoding.cKeyPageCount];
for (int i = 0; i < encoding.cKeyPageCount; i++)
{
encoding.aHeaders[i].firstHash = bin.Read <MD5Hash>();
encoding.aHeaders[i].checksum = bin.Read <MD5Hash>();
}
}
else
{
bin.BaseStream.Position += encoding.cKeyPageCount * 32;
}
var tableAstart = bin.BaseStream.Position;
Dictionary <MD5Hash, EncodingFileEntry> entries = new Dictionary <MD5Hash, EncodingFileEntry>(new MD5HashComparer());
for (int i = 0; i < encoding.cKeyPageCount; i++)
{
byte keysCount;
while ((keysCount = bin.ReadByte()) != 0)
{
EncodingFileEntry entry = new EncodingFileEntry()
{
size = bin.ReadInt40BE()
};
var cKey = bin.Read <MD5Hash>();
// @TODO add support for multiple encoding keys
for (int key = 0; key < keysCount; key++)
{
if (key == 0)
{
entry.eKey = bin.Read <MD5Hash>();
}
else
{
bin.ReadBytes(16);
}
}
entries.Add(cKey, entry);
try
{
encodingCacheLock.EnterUpgradeableReadLock();
if (!encodingDictionary.ContainsKey(cKey))
{
try
{
encodingCacheLock.EnterWriteLock();
encodingDictionary.Add(cKey, entry.eKey);
}
finally
{
encodingCacheLock.ExitWriteLock();
}
}
}
finally
{
encodingCacheLock.ExitUpgradeableReadLock();
}
}
var remaining = 4096 - ((bin.BaseStream.Position - tableAstart) % 4096);
if (remaining > 0)
{
bin.BaseStream.Position += remaining;
}
}
encoding.aEntries = entries;
if (!parseTableB)
{
return(encoding);
}
/* Table B */
if (checkStuff)
{
encoding.bHeaders = new EncodingHeaderEntry[encoding.eKeyPageCount];
for (int i = 0; i < encoding.eKeyPageCount; i++)
{
encoding.bHeaders[i].firstHash = bin.Read <MD5Hash>();
encoding.bHeaders[i].checksum = bin.Read <MD5Hash>();
}
}
else
{
bin.BaseStream.Position += encoding.eKeyPageCount * 32;
}
var tableBstart = bin.BaseStream.Position;
List <EncodingFileDescEntry> b_entries = new List <EncodingFileDescEntry>();
while (bin.BaseStream.Position < tableBstart + 4096 * encoding.eKeyPageCount)
{
var remaining = 4096 - (bin.BaseStream.Position - tableBstart) % 4096;
if (remaining < 25)
{
bin.BaseStream.Position += remaining;
continue;
}
EncodingFileDescEntry entry = new EncodingFileDescEntry()
{
key = bin.Read <MD5Hash>(),
stringIndex = bin.ReadUInt32(true),
compressedSize = bin.ReadUInt40(true)
};
if (entry.stringIndex == uint.MaxValue)
{
break;
}
b_entries.Add(entry);
}
encoding.bEntries = b_entries.ToArray();
}
return(encoding);
}
public static async Task <CDNConfigFile> GetCDNConfig(string hash)
{
var cdnConfig = new CDNConfigFile();
string content = System.Text.Encoding.UTF8.GetString(await CDNCache.Get("config", hash));
var cdnConfigLines = content.Split(new string[] { "\n" }, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < cdnConfigLines.Count(); i++)
{
if (cdnConfigLines[i].StartsWith("#") || cdnConfigLines[i].Length == 0)
{
continue;
}
var cols = cdnConfigLines[i].Split(new string[] { " = " }, StringSplitOptions.RemoveEmptyEntries);
switch (cols[0])
{
case "archives":
var archives = cols[1].Split(' ');
cdnConfig.archives = new MD5Hash[archives.Length];
for (var j = 0; j < archives.Length; j++)
{
cdnConfig.archives[j] = archives[j].ToByteArray().ToMD5();
}
break;
case "archive-group":
cdnConfig.archiveGroup = cols[1];
break;
case "patch-archives":
if (cols.Length > 1)
{
var patchArchives = cols[1].Split(' ');
cdnConfig.patchArchives = new MD5Hash[patchArchives.Length];
for (var j = 0; j < patchArchives.Length; j++)
{
cdnConfig.patchArchives[j] = patchArchives[j].ToByteArray().ToMD5();
}
}
break;
case "patch-archive-group":
cdnConfig.patchArchiveGroup = cols[1];
break;
case "builds":
var builds = cols[1].Split(' ');
cdnConfig.builds = builds;
break;
case "file-index":
cdnConfig.fileIndex = cols[1];
break;
case "file-index-size":
cdnConfig.fileIndexSize = cols[1];
break;
case "patch-file-index":
cdnConfig.patchFileIndex = cols[1];
break;
case "patch-file-index-size":
cdnConfig.patchFileIndexSize = cols[1];
break;
case "archives-index-size":
case "patch-archives-index-size":
break;
default:
Logger.WriteLine("!!!!!!!! Unknown cdnconfig variable '" + cols[0] + "'");
break;
}
}
return(cdnConfig);
}
public static async Task <BuildConfigFile> GetBuildConfig(string hash)
{
var buildConfig = new BuildConfigFile();
string content = System.Text.Encoding.UTF8.GetString(await CDNCache.Get("config", hash));
if (string.IsNullOrEmpty(content) || !content.StartsWith("# Build"))
{
Console.WriteLine("Error reading build config!");
return(buildConfig);
}
var lines = content.Split(new string[] { "\n" }, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < lines.Count(); i++)
{
if (lines[i].StartsWith("#") || lines[i].Length == 0)
{
continue;
}
var cols = lines[i].Split(new string[] { " = " }, StringSplitOptions.RemoveEmptyEntries);
switch (cols[0])
{
case "root":
buildConfig.root = cols[1].ToByteArray().ToMD5();
break;
case "download":
var downloadSplit = cols[1].Split(' ');
buildConfig.download = new MD5Hash[downloadSplit.Length];
buildConfig.download[0] = downloadSplit[0].ToByteArray().ToMD5();
if (downloadSplit.Length > 1)
{
buildConfig.download[1] = downloadSplit[1].ToByteArray().ToMD5();
}
break;
case "install":
var installSplit = cols[1].Split(' ');
buildConfig.install = new MD5Hash[installSplit.Length];
buildConfig.install[0] = installSplit[0].ToByteArray().ToMD5();
if (installSplit.Length > 1)
{
buildConfig.install[1] = installSplit[1].ToByteArray().ToMD5();
}
break;
case "encoding":
var encodingSplit = cols[1].Split(' ');
buildConfig.encoding = new MD5Hash[encodingSplit.Length];
buildConfig.encoding[0] = encodingSplit[0].ToByteArray().ToMD5();
if (encodingSplit.Length > 1)
{
buildConfig.encoding[1] = encodingSplit[1].ToByteArray().ToMD5();
}
break;
case "encoding-size":
var encodingSize = cols[1].Split(' ');
buildConfig.encodingSize = encodingSize;
break;
case "size":
buildConfig.size = cols[1].Split(' ');
break;
case "size-size":
buildConfig.sizeSize = cols[1].Split(' ');
break;
case "build-name":
buildConfig.buildName = cols[1];
break;
case "build-playbuild-installer":
buildConfig.buildPlaybuildInstaller = cols[1];
break;
case "build-product":
buildConfig.buildProduct = cols[1];
break;
case "build-uid":
buildConfig.buildUid = cols[1];
break;
case "patch":
buildConfig.patch = cols[1].ToByteArray().ToMD5();
break;
case "patch-size":
buildConfig.patchSize = cols[1];
break;
case "patch-config":
buildConfig.patchConfig = cols[1].ToByteArray().ToMD5();
break;
case "build-branch": // Overwatch
buildConfig.buildBranch = cols[1];
break;
case "build-num": // Agent
case "build-number": // Overwatch
case "build-version": // Catalog
buildConfig.buildNumber = cols[1];
break;
case "build-attributes": // Agent
buildConfig.buildAttributes = cols[1];
break;
case "build-comments": // D3
buildConfig.buildComments = cols[1];
break;
case "build-creator": // D3
buildConfig.buildCreator = cols[1];
break;
case "build-fixed-hash": // S2
buildConfig.buildFixedHash = cols[1];
break;
case "build-replay-hash": // S2
buildConfig.buildReplayHash = cols[1];
break;
case "build-t1-manifest-version":
buildConfig.buildManifestVersion = cols[1];
break;
case "install-size":
buildConfig.installSize = cols[1].Split(' ');
break;
case "download-size":
buildConfig.downloadSize = cols[1].Split(' ');
break;
case "build-partial-priority":
case "partial-priority":
buildConfig.partialPriority = cols[1];
break;
case "partial-priority-size":
buildConfig.partialPrioritySize = cols[1];
break;
default:
Logger.WriteLine("!!!!!!!! Unknown buildconfig variable '" + cols[0] + "'");
break;
}
}
return(buildConfig);
}
public static async Task <BuildConfigFile> GetBuildConfig(string hash)
{
var buildConfig = new BuildConfigFile();
string content = System.Text.Encoding.UTF8.GetString(await CDNCache.Get("config", hash));
if (string.IsNullOrEmpty(content) || !content.StartsWith("# Build"))
{
Console.WriteLine("Error reading build config!");
return(buildConfig);
}
var lines = content.Split(new string[] { "\n" }, StringSplitOptions.RemoveEmptyEntries);
for (var i = 0; i < lines.Count(); i++)
{
if (lines[i].StartsWith("#") || lines[i].Length == 0)
{
continue;
}
var cols = lines[i].Split(new string[] { " = " }, StringSplitOptions.RemoveEmptyEntries);
switch (cols[0])
{
case "root":
buildConfig.root = cols[1].ToByteArray().ToMD5();
break;
case "install":
var installSplit = cols[1].Split(' ');
buildConfig.install = new MD5Hash[installSplit.Length];
buildConfig.install[0] = installSplit[0].ToByteArray().ToMD5();
if (installSplit.Length > 1)
{
buildConfig.install[1] = installSplit[1].ToByteArray().ToMD5();
}
break;
case "encoding":
var encodingSplit = cols[1].Split(' ');
buildConfig.encoding = new MD5Hash[encodingSplit.Length];
buildConfig.encoding[0] = encodingSplit[0].ToByteArray().ToMD5();
if (encodingSplit.Length > 1)
{
buildConfig.encoding[1] = encodingSplit[1].ToByteArray().ToMD5();
}
break;
case "encoding-size":
var encodingSize = cols[1].Split(' ');
buildConfig.encodingSize = encodingSize;
break;
default:
break;
}
}
return(buildConfig);
}
