public void AddEntry(CASResult blte) { var entry = new LocalIndexEntry() { Archive = blte.Archive, Key = blte.EKey.Value.Take(9).ToArray(), Offset = blte.Offset, Size = blte.CompressedSize }; var idx = LocalIndices.First(x => x.BucketIndex == GetBucket(entry.Key)); var existing = idx.Entries.FirstOrDefault(x => x.Key.SequenceEqual(entry.Key)); // check for existing if (existing != null) { existing = entry; } else { idx.Entries.Add(entry); } idx.Changed = true; }
public void AddEntry(CASResult blte) { if (blte == null) { return; } // create the entry var entry = new EncodingCEKeyPageTable() { DecompressedSize = blte.DecompressedSize, CKey = blte.CEKey, }; entry.EKeys.Add(blte.EKey); if (CEKeys.ContainsKey(blte.CEKey)) // check if it exists { var existing = CEKeys[blte.CEKey]; if (EKeys.ContainsKey(existing.EKeys[0])) // remove old layout { EKeys.Remove(existing.EKeys[0]); } existing.EKeys[0] = blte.EKey; // update existing entry } else { CEKeys.Add(entry.CKey, entry); // new entry } AddLayoutEntry(blte); }
public void AddEntry(CASResult blte) { if (blte == null) { return; } // create the entry var entry = new EncodingEntry() { DecompressedSize = blte.DecompressedSize, Hash = blte.DataHash, }; entry.Keys.Add(blte.Hash); if (Data.ContainsKey(blte.DataHash)) // check if it exists { var existing = Data[blte.DataHash]; if (Layout.ContainsKey(existing.Keys[0])) // remove old layout { Layout.Remove(existing.Keys[0]); } existing.Keys[0] = blte.Hash; // update existing entry } else { Data.Add(entry.Hash, entry); // new entry } AddLayoutEntry(blte); }
public void AddEntry(string path, CASResult file) { ulong hash = new Jenkins96().ComputeHash(path); bool found = false; // check to see if we're overwriting an existing entry foreach (var root in Chunks) { if (!root.LocaleFlags.HasFlag(locale) && root != GlobalRoot) // skip incorrect locales and non-global roots { continue; } var index = root.Entries.FindIndex(x => x.Hash == hash); if (index >= 0) { RootEntry entry = new RootEntry() { MD5 = file.DataHash, Hash = hash, Path = path, FileDataId = root.Entries[index].FileDataId, FileDataIdOffset = root.Entries[index].FileDataIdOffset }; root.Entries[index] = entry; // update found = true; // max id check just to be safe if (root == GlobalRoot) { maxId = Math.Max(entry.FileDataId, maxId); // update max id } CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash)); } } // must be a new file, add it to the global root if (!found) { RootEntry entry = new RootEntry() { MD5 = file.DataHash, FileDataIdOffset = 0, Hash = hash, Path = path }; var cache = CASContainer.Settings.Cache.Entries.FirstOrDefault(x => x.Path == path); if (cache?.Path != path) // get cache id { entry.FileDataId = Math.Max(maxId + 1, minimumId); // calculate the Id } GlobalRoot.Entries.Add(entry); // add new maxId = Math.Max(entry.FileDataId, maxId); // Update max id CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash)); } }
private void AddLayoutEntry(CASResult blte) { if (EKeys.ContainsKey(blte.EKey)) { EKeys.Remove(blte.EKey); } // generate ESpecString string ESpecString; uint size = blte.CompressedSize - 30; // the below suffices and is technically correct // however this could be more compliant https://wowdev.wiki/CASC#Encoding_Specification_.28ESpec.29 if (blte.CEKey == CASContainer.BuildConfig.GetKey("root")) // root is always z { ESpecString = "z"; } else if (size >= 1024 * 256) // 256K* seems to be the max { ESpecString = "b:{256K*=z}"; } else if (size > 1024) { ESpecString = "b:{" + (int)Math.Floor(size / 1024d) + "K*=z}"; // closest floored KB } else { ESpecString = "b:{" + size + "*=z}"; // actual B size } // string index int stridx = ESpecStringTable.IndexOf(ESpecString); if (stridx == -1) { stridx = ESpecStringTable.Count - 2; // ignore the 0 byte ESpecStringTable.Insert(stridx, ESpecString); } // create the entry var entry = new EncodingEKeyPageTable() { FileSize = size, EKey = blte.EKey, ESpecStringIndex = (uint)stridx }; EKeys.Add(entry.EKey, entry); }
public void AddEntry(CASResult blte) { if (CASContainer.EncodingHandler.Layout.ContainsKey(blte.Hash)) // skip existing { return; } var entry = new DownloadEntry() { Hash = blte.Hash, FileSize = blte.CompressedSize - 30, UnknownData = new byte[4], Stage = (byte)(blte.HighPriority ? 0 : 1) }; int index = endofStageIndex[entry.Stage]; if (index >= 0) { if (entry.Stage == 0) { endofStageIndex[0]++; } endofStageIndex[1]++; Entries.Insert(index, entry); foreach (var tag in Tags) { if (tag.Name != "Alternate") { tag.BitMask.Insert(index, true); } } } else { Entries.Add(entry); foreach (var tag in Tags) { if (tag.Name != "Alternate") { tag.BitMask.Add(true); } } } }
public CASResult Write() { FixOffsets(); using (var md5 = MD5.Create()) using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms)) { // write each chunk foreach (var c in Chunks) { bw.Write((uint)c.Entries.Count); bw.Write((uint)c.ContentFlags); bw.Write((uint)c.LocaleFlags); foreach (var e in c.Entries) { bw.Write(e.FileDataIdOffset); } foreach (var e in c.Entries) { bw.Write(e.CEKey.Value); bw.Write(e.NameHash); } } // create CASCFile CASFile entry = new CASFile(ms.ToArray(), encodingMap.Type, encodingMap.CompressionLevel); // save and update Build Config CASResult res = DataHandler.Write(WriteMode.CDN, entry); res.CEKey = new MD5Hash(md5.ComputeHash(ms.ToArray())); res.HighPriority = true; CASContainer.BuildConfig.Set("root", res.CEKey.ToString()); CASContainer.Logger.LogInformation($"Root: EKey: {res.EKey} CEKey: {res.CEKey}"); // cache Root Hash CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry() { CEKey = res.CEKey, EKey = res.EKey, Path = "__ROOT__" }); return(res); } }
public void AddEntry(string path, CASResult file) { var cache = CASContainer.Settings.Cache; ulong namehash = new Jenkins96().ComputeHash(path); var entries = Chunks .FindAll(chunk => chunk.LocaleFlags.HasFlag(locale)) // Select locales that match selected locale .SelectMany(chunk => chunk.Entries) // Flatten the array to get all entries within all matching chunks .Where(e => e.NameHash == namehash); if (entries.Count() == 0) { // New file, we need to create an entry for it var cached = cache.Entries.FirstOrDefault(x => x.Path == path); var fileDataId = Math.Max(maxId + 1, minimumId); if (cached != null) { fileDataId = cached.FileDataId; } var entry = new RootEntry() { CEKey = file.CEKey, FileDataId = fileDataId, FileDataIdOffset = 0, NameHash = namehash, Path = path }; GlobalRoot.Entries.Add(entry); // Insert into the Global Root maxId = Math.Max(entry.FileDataId, maxId); // Update the max id cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); // If not done, sometimes files will not be added. } else { // Existing file, we just have to update the data hash foreach (var entry in entries) { entry.CEKey = file.CEKey; entry.Path = path; cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); } } }
private void AddLayoutEntry(CASResult blte) { if (Layout.ContainsKey(blte.Hash)) { Layout.Remove(blte.Hash); } // get layout string string layoutString; uint size = blte.CompressedSize - 30; if (blte.DataHash == CASContainer.BuildConfig.GetKey("root")) // root is always z { layoutString = "z"; } else if (size >= 1024 * 256) // 256K* seems to be the max { layoutString = "b:{256K*=z}"; } else if (size > 1024) { layoutString = "b:{" + (int)Math.Floor(size / 1024d) + "K*=z}"; // closest floored KB } else { layoutString = "b:{" + size + "*=z}"; // actual B size } // string index int stridx = LayoutStringTable.IndexOf(layoutString); if (stridx == -1) { stridx = LayoutStringTable.Count - 2; // ignore the 0 byte LayoutStringTable.Insert(stridx, layoutString); } // create the entry var entry = new EncodingLayout() { Size = size, Hash = blte.Hash, StringIndex = (uint)stridx }; Layout.Add(entry.Hash, entry); }
public void AddEntry(CASResult blte) { if (CASContainer.EncodingHandler.EKeys.ContainsKey(blte.EKey)) // skip existing { return; } var entry = new DownloadEntry() { EKey = blte.EKey, FileSize = blte.CompressedSize - 30, Flags = new DownloadFlags[Header.NumFlags], Priority = (byte)(blte.HighPriority ? 0 : 1) }; if (Header.HasChecksum != 0) { //entry.Checksum = } int index = endofStageIndex[entry.Priority]; if (index >= 0) { endofStageIndex[entry.Priority]++; Entries.Insert(index, entry); foreach (var tag in Tags) { tag.BitMask.Insert(index, tag.Name != "Alternate"); } } else { Entries.Add(entry); foreach (var tag in Tags) { tag.BitMask.Add(tag.Name != "Alternate"); } } }
public CASResult Write(List <CASResult> newentries) { if (!NeedsWrite(newentries)) { return(null); } byte[][] entries = new byte[EncodingMap.Length][]; CASFile[] files = new CASFile[EncodingMap.Length]; // header using (var ms = new MemoryStream()) using (var bw = new BinaryWriter(ms)) { bw.Write(Header.Magic); bw.Write(Header.Version); bw.Write(Header.HashSize); bw.WriteUInt16BE((ushort)Tags.Count); bw.WriteUInt32BE((uint)InstallData.Count); foreach (var tag in Tags) { bw.Write(Encoding.UTF8.GetBytes(tag.Name)); bw.Write((byte)0); bw.WriteUInt16BE(tag.Type); bw.Write(tag.BitMask.ToByteArray()); } entries[0] = ms.ToArray(); files[0] = new CASFile(entries[0], EncodingMap[0].Type, EncodingMap[0].CompressionLevel); } // entries using (var ms = new MemoryStream()) using (var bw = new BinaryWriter(ms)) { foreach (var entry in InstallData) { bw.Write(Encoding.UTF8.GetBytes(entry.Name)); bw.Write((byte)0); bw.Write(entry.CEKey.Value); bw.WriteUInt32BE(entry.Size); } entries[1] = ms.ToArray(); files[1] = new CASFile(entries[1], EncodingMap[1].Type, EncodingMap[1].CompressionLevel); } // write CASResult res = DataHandler.Write(WriteMode.CDN, files); using (var md5 = MD5.Create()) res.CEKey = new MD5Hash(md5.ComputeHash(entries.SelectMany(x => x).ToArray())); Console.WriteLine($"Install: EKey: {res.EKey} CEKey: {res.CEKey}"); CASContainer.BuildConfig.Set("install-size", res.DecompressedSize.ToString()); CASContainer.BuildConfig.Set("install-size", (res.CompressedSize - 30).ToString(), 1); // BLTE size minus header CASContainer.BuildConfig.Set("install", res.CEKey.ToString()); CASContainer.BuildConfig.Set("install", res.EKey.ToString(), 1); Array.Resize(ref entries, 0); Array.Resize(ref files, 0); return(res); }
public CASResult Write() { byte[][] entries = new byte[EncodingMap.Length][]; CASFile[] files = new CASFile[EncodingMap.Length]; // ESpecStringTable 1 entries[1] = Encoding.UTF8.GetBytes(string.Join("\0", ESpecStringTable)); files[1] = new CASFile(entries[1], EncodingMap[1].Type, EncodingMap[1].CompressionLevel); // CEKeysPageTable Data 3 using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms)) { long pos = 0; foreach (var entry in CEKeys.Values) { if (pos + entry.EntrySize > CHUNK_SIZE) { bw.Write(new byte[CHUNK_SIZE - pos]); // pad to chunk size pos = 0; } bw.Write((ushort)entry.EKeys.Count); bw.WriteUInt32BE(entry.DecompressedSize); bw.Write(entry.CKey.Value); for (int i = 0; i < entry.EKeys.Count; i++) { bw.Write(entry.EKeys[i].Value); } pos += entry.EntrySize; } bw.Write(new byte[CHUNK_SIZE - pos]); // final padding entries[3] = ms.ToArray(); files[3] = new CASFile(entries[3], EncodingMap[3].Type, EncodingMap[3].CompressionLevel); } // EKeysPageTable Data 5 using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms)) { long pos = 0; foreach (var entry in EKeys.Values) { if (pos + entry.EntrySize > CHUNK_SIZE) { bw.Write(new byte[CHUNK_SIZE - pos]); // pad to chunk size pos = 0; } bw.Write(entry.EKey.Value); bw.WriteUInt32BE(entry.ESpecStringIndex); bw.WriteUInt40BE(entry.FileSize); pos += entry.EntrySize; } // EOF flag bw.Write(new byte[16]); // empty hash bw.Write(0xFFFFFFFF); // flag pos += 16 + 4; bw.Write(new byte[CHUNK_SIZE - pos]); // final padding entries[5] = ms.ToArray(); files[5] = new CASFile(entries[5], EncodingMap[5].Type, EncodingMap[5].CompressionLevel); } // CEKeysPageTable lookup 2 using (var md5 = MD5.Create()) using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms)) { int chunks = entries[3].Length / CHUNK_SIZE; for (int i = 0; i < chunks; i++) { byte[] chunk = new byte[CHUNK_SIZE]; Buffer.BlockCopy(entries[3], (i * CHUNK_SIZE), chunk, 0, CHUNK_SIZE); bw.Write(chunk, 6, 16); // first entry hash bw.Write(md5.ComputeHash(chunk)); // md5 of chunk } entries[2] = ms.ToArray(); files[2] = new CASFile(entries[2], EncodingMap[2].Type, EncodingMap[2].CompressionLevel); } // EKeysPageTable lookup 4 using (var md5 = MD5.Create()) using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms)) { int chunks = entries[5].Length / CHUNK_SIZE; for (int i = 0; i < chunks; i++) { byte[] chunk = new byte[CHUNK_SIZE]; Buffer.BlockCopy(entries[5], (i * CHUNK_SIZE), chunk, 0, CHUNK_SIZE); bw.Write(chunk, 0, 16); // first entry hash bw.Write(md5.ComputeHash(chunk)); // md5 of chunk } entries[4] = ms.ToArray(); files[4] = new CASFile(entries[4], EncodingMap[4].Type, EncodingMap[4].CompressionLevel); } // Encoding Header 0 using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms)) { bw.Write(Header.Magic); bw.Write(Header.Version); bw.Write(Header.ChecksumSizeC); bw.Write(Header.ChecksumSizeE); bw.Write(Header.PageSizeCEKey); bw.Write(Header.PageSizeEKey); bw.WriteUInt32BE((uint)entries[2].Length / 32); bw.WriteUInt32BE((uint)entries[4].Length / 32); bw.Write(Header.Unknown_x11); bw.WriteUInt32BE((uint)Encoding.UTF8.GetByteCount(string.Join("\0", ESpecStringTable))); entries[0] = ms.ToArray(); files[0] = new CASFile(entries[0], EncodingMap[0].Type, EncodingMap[0].CompressionLevel); } // Encoding's own ESpecStringTable 6 entries[6] = GetStringTable(entries.Select(x => x.Length)); files[6] = new CASFile(entries[6], EncodingMap[6].Type, EncodingMap[6].CompressionLevel); //Write CASResult res = DataHandler.Write(WriteMode.CDN, files); using (var md5 = MD5.Create()) res.CEKey = new MD5Hash(md5.ComputeHash(entries.SelectMany(x => x).ToArray())); CASContainer.Logger.LogInformation($"Encoding: EKey: {res.EKey} CEKey: {res.CEKey}"); CASContainer.BuildConfig.Set("encoding-size", res.DecompressedSize.ToString()); CASContainer.BuildConfig.Set("encoding-size", (res.CompressedSize - 30).ToString(), 1); // BLTE size minus header CASContainer.BuildConfig.Set("encoding", res.CEKey.ToString()); CASContainer.BuildConfig.Set("encoding", res.EKey.ToString(), 1); Array.Resize(ref entries, 0); Array.Resize(ref files, 0); entries = null; files = null; // cache Encoding Hash CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry() { CEKey = res.CEKey, EKey = res.EKey, Path = "__ENCODING__" }); return(res); }
public static CASResult Write(WriteMode mode, params CASFile[] entries) { CASContainer.Logger.LogInformation("Writing data..."); var path = Path.Combine(CASContainer.Settings.OutputPath, "Data", "data"); string filename = "dummy.000"; // calculate local data file if (mode.HasFlag(WriteMode.Data)) { Directory.CreateDirectory(path); long requiredBytes = entries.Sum(x => x.Data.Length) + 38 + (entries.Count() > 1 ? 5 : 0) + (entries.Count() * 24); if (mode.HasFlag(WriteMode.Data)) { filename = GetDataFile(requiredBytes); } } using (var md5 = MD5.Create()) using (MemoryStream ms = new MemoryStream()) using (BinaryWriter bw = new BinaryWriter(ms, Encoding.ASCII)) { bw.Seek(0, SeekOrigin.End); long posStart = bw.BaseStream.Position; uint headersize = entries.Length == 1 ? 0 : 24 * (uint)entries.Length + 12; // Archive Header bw.Write(new byte[16]); // MD5 hash bw.Write((uint)0); // Size bw.Write(new byte[0xA]); // Unknown // Header bw.Write(BLTEStream.BLTE_MAGIC); bw.WriteUInt32BE(headersize); // Chunkinfo if (headersize > 0) { bw.Write((byte)15); // Flags bw.WriteUInt24BE((uint)entries.Count()); // Entries foreach (var entry in entries) { bw.WriteUInt32BE(entry.CompressedSize); bw.WriteUInt32BE(entry.DecompressedSize); bw.Write(md5.ComputeHash(entry.Data)); // Checksum } } // Write data foreach (var entry in entries) { bw.Write(entry.Data); Array.Resize(ref entry.Data, 0); } // Compute header hash bw.BaseStream.Position = posStart + 30; byte[] buffer = new byte[(headersize == 0 ? bw.BaseStream.Length - bw.BaseStream.Position : headersize)]; bw.BaseStream.Read(buffer, 0, buffer.Length); var hash = md5.ComputeHash(buffer); CASResult blte = new CASResult() { DecompressedSize = (uint)entries.Sum(x => x.DecompressedSize), CompressedSize = (uint)(bw.BaseStream.Length - posStart), EKey = new MD5Hash(hash) }; bw.BaseStream.Position = posStart; bw.Write(hash.Reverse().ToArray()); // Write Hash bw.Write(blte.CompressedSize); // Write Length // Update .data file if (mode.HasFlag(WriteMode.Data)) { Directory.CreateDirectory(path); using (FileStream fs = new FileStream(Path.Combine(path, filename), FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.Seek(0, SeekOrigin.End); blte.Offset = (ulong)fs.Position; ms.Position = 0; ms.CopyTo(fs); fs.Flush(); } } // Output raw if (mode.HasFlag(WriteMode.CDN)) { blte.OutPath = Path.Combine(CASContainer.Settings.OutputPath, Helper.GetCDNPath(blte.EKey.ToString(), "data")); using (FileStream fs = new FileStream(blte.OutPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite)) { ms.Position = 30; // Skip header ms.CopyTo(fs); fs.Flush(); } } return(blte); } }
public CASResult Write() { byte[][] entries = new byte[EncodingMap.Length][]; CASFile[] files = new CASFile[EncodingMap.Length]; // header using (var ms = new MemoryStream()) using (var bw = new BinaryWriter(ms)) { bw.Write(Header.Header); bw.Write(Header.Version); bw.Write(Header.ChecksumSize); bw.Write(Header.HasChecksum); bw.WriteUInt32BE((uint)Entries.Count); bw.WriteUInt16BE((ushort)Tags.Count); if (Header.Version >= 2) { bw.Write(Header.NumFlags); } if (Header.Version >= 3) { // TODO //bw.Write(Header.BasePriority); //bw.Write(Header.Unknown_0D); } entries[0] = ms.ToArray(); files[0] = new CASFile(entries[0], EncodingMap[0].Type, EncodingMap[0].CompressionLevel); } // files using (var ms = new MemoryStream()) using (var bw = new BinaryWriter(ms)) { foreach (var entry in Entries) { bw.Write(entry.EKey.Value); bw.WriteUInt40BE(entry.FileSize); bw.Write(entry.Priority); if (Header.HasChecksum != 0) { bw.WriteUInt32BE(entry.Checksum); } if (Header.Version >= 2) { bw.Write((byte[])(object)entry.Flags); } } entries[1] = ms.ToArray(); files[1] = new CASFile(entries[1], EncodingMap[1].Type, EncodingMap[1].CompressionLevel); } // tags using (var ms = new MemoryStream()) using (var bw = new BinaryWriter(ms)) { foreach (var tag in Tags) { bw.Write(Encoding.UTF8.GetBytes(tag.Name)); bw.Write((byte)0); bw.WriteUInt16BE(tag.Type); bw.Write(tag.BitMask.ToByteArray()); tag.BitMask.Clear(); } entries[2] = ms.ToArray(); files[2] = new CASFile(entries[2], EncodingMap[2].Type, EncodingMap[2].CompressionLevel); } // write CASResult res = DataHandler.Write(WriteMode.CDN, files); using (var md5 = MD5.Create()) res.CEKey = new MD5Hash(md5.ComputeHash(entries.SelectMany(x => x).ToArray())); File.Delete(Path.Combine(CASContainer.Settings.OutputPath, CASContainer.BuildConfig["download"][0])); CASContainer.Logger.LogInformation($"Download: EKey: {res.EKey} CEKey: {res.CEKey}"); CASContainer.BuildConfig.Set("download-size", res.DecompressedSize.ToString()); CASContainer.BuildConfig.Set("download-size", (res.CompressedSize - 30).ToString(), 1); CASContainer.BuildConfig.Set("download", res.CEKey.ToString()); CASContainer.BuildConfig.Set("download", res.EKey.ToString(), 1); Array.Resize(ref entries, 0); Array.Resize(ref files, 0); entries = null; files = null; return(res); }
public int Compare(CASResult x, CASResult y) => Compare(x.Hash.Value, y.Hash.Value);
public int Compare(CASResult x, CASResult y) => Compare(x.EKey.Value, y.EKey.Value);