static Jenkins96 HandleLine(string line, ParallelLoopState state, Jenkins96 hasher) { if (string.IsNullOrWhiteSpace(line)) { return(hasher); } if (line.IndexOfAny(pathInvalidChars) != -1) { return(hasher); } line = line.Replace('/', '\\'); //string extLower = Path.GetExtension(line).ToLower(); //if (extLower == ".mdx" || extLower == ".mdl") //{ // line = Path.ChangeExtension(line, ".m2"); // extLower = ".m2"; //} ulong hash = hasher.ComputeHash(line); if (!CheckName(hash, line)) { return(hasher); } processed[hash] = true; return(hasher); }
static void Main(string[] args) { Console.WriteLine("Hashing devices found ::"); foreach (var AcceleratorDevice in AcceleratorDevice.All) { Console.WriteLine($" {AcceleratorDevice}"); } Console.WriteLine(""); if (args == null || args.Length == 0) { throw new ArgumentException("No arguments supplied"); } IHash hash; switch (args[0].ToLowerInvariant()) { case "salsa": hash = new Salsa20(); break; case "jenkins": hash = new Jenkins96(); break; case "benchmark": BenchmarkJenkins(args); return; case "wordlist": hash = new Wordlist(); break; case "variation": hash = new VarientGen(); break; case "mix": hash = new MixMatch(); break; case "markov": hash = new MarkovChain(); break; default: throw new ArgumentException("Invalid hash type"); } // override console exit event CleanExitHandler.Attach(); hash.LoadParameters(args); hash.Start(); Console.ReadLine(); }
public void AddEntry(string path, CASResult file) { ulong hash = new Jenkins96().ComputeHash(path); bool found = false; // check to see if we're overwriting an existing entry foreach (var root in Chunks) { if (!root.LocaleFlags.HasFlag(locale) && root != GlobalRoot) // skip incorrect locales and non-global roots { continue; } var index = root.Entries.FindIndex(x => x.Hash == hash); if (index >= 0) { RootEntry entry = new RootEntry() { MD5 = file.DataHash, Hash = hash, Path = path, FileDataId = root.Entries[index].FileDataId, FileDataIdOffset = root.Entries[index].FileDataIdOffset }; root.Entries[index] = entry; // update found = true; // max id check just to be safe if (root == GlobalRoot) { maxId = Math.Max(entry.FileDataId, maxId); // update max id } CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash)); } } // must be a new file, add it to the global root if (!found) { RootEntry entry = new RootEntry() { MD5 = file.DataHash, FileDataIdOffset = 0, Hash = hash, Path = path }; var cache = CASContainer.Settings.Cache.Entries.FirstOrDefault(x => x.Path == path); if (cache?.Path != path) // get cache id { entry.FileDataId = Math.Max(maxId + 1, minimumId); // calculate the Id } GlobalRoot.Entries.Add(entry); // add new maxId = Math.Max(entry.FileDataId, maxId); // Update max id CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash)); } }
static void LocalFinal(Jenkins96 local) { int secs = (int)(DateTime.Now - startTime).TotalSeconds; if (secs > 0) { currentSpeed = (int)(hashCount / secs); } ConsoleProgressBar.DrawProgressBar(currentGenerator.Percent, 71, '#', false, currentSpeed); }
static void BenchmarkJenkins(params string[] args) { int perms = 5; if (args != null && args.Length > 1 && !int.TryParse(args[1], out perms)) { perms = 5; } perms = Math.Min(Math.Max(perms, 1), 9); // clamp to 1 - 9 char[] filename = "interface/cinematics/legion_dh2.mp3".ToCharArray(); // build the dummy template for (int i = 0; i < perms; i++) { filename[30 - i] = '%'; } Console.WriteLine($"Benchmarking Jenkins @ {perms} permutations (39^{perms}) :: "); List <string> types = new List <string>() { "gpu", "cpu", "all" }; // calculate the available devices if (!AcceleratorDevice.HasCPU) { types.Remove("cpu"); } if (!AcceleratorDevice.HasGPU) { types.Remove("gpu"); } if (types.Count == 2 && AcceleratorDevice.All.Length == 1) { types.Remove("all"); } Jenkins96 jenkins = new Jenkins96(); foreach (var type in types) { jenkins.LoadTestParameters(type, new string(filename)); Console.WriteLine(""); Console.WriteLine($"-- {type.ToUpper()} Test --"); jenkins.Start(); } Console.ReadLine(); }
public void AddEntry(string path, CASResult file) { var cache = CASContainer.Settings.Cache; ulong namehash = new Jenkins96().ComputeHash(path); var entries = Chunks .FindAll(chunk => chunk.LocaleFlags.HasFlag(locale)) // Select locales that match selected locale .SelectMany(chunk => chunk.Entries) // Flatten the array to get all entries within all matching chunks .Where(e => e.NameHash == namehash); if (entries.Count() == 0) { // New file, we need to create an entry for it var cached = cache.Entries.FirstOrDefault(x => x.Path == path); var fileDataId = Math.Max(maxId + 1, minimumId); if (cached != null) { fileDataId = cached.FileDataId; } var entry = new RootEntry() { CEKey = file.CEKey, FileDataId = fileDataId, FileDataIdOffset = 0, NameHash = namehash, Path = path }; GlobalRoot.Entries.Add(entry); // Insert into the Global Root maxId = Math.Max(entry.FileDataId, maxId); // Update the max id cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); // If not done, sometimes files will not be added. } else { // Existing file, we just have to update the data hash foreach (var entry in entries) { entry.CEKey = file.CEKey; entry.Path = path; cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); } } }
public void RemoveFile(string path) { ulong hash = new Jenkins96().ComputeHash(path); foreach (var root in Chunks) { var entries = root.Entries.Where(x => x.NameHash == hash).ToArray(); // should only ever be one but just incase foreach (var entry in entries) { if (CASContainer.EncodingHandler.CEKeys.TryGetValue(entry.CEKey, out EncodingCEKeyPageTable enc)) { CASContainer.DownloadHandler?.RemoveEntry(enc.EKeys[0]); // remove from download CASContainer.CDNIndexHandler?.RemoveEntry(enc.EKeys[0]); // remove from cdn index } root.Entries.Remove(entry); CASContainer.Settings.Cache?.Remove(path); } } }
static void Main(string[] args) { Console.WriteLine("Hashing devices found ::"); foreach (var AcceleratorDevice in AcceleratorDevice.All) { Console.WriteLine($" {AcceleratorDevice}"); } Console.WriteLine(""); if (args == null || args.Length == 0) { throw new ArgumentException("No arguments supplied"); } IHash hash; switch (args[0].ToLowerInvariant()) { case "salsa": hash = new Salsa20(); break; case "jenkins": hash = new Jenkins96(); break; case "benchmark": BenchmarkJenkins(args); return; default: throw new ArgumentException("Invalid hash type"); } hash.LoadParameters(args); hash.Start(); }
public void RenameFile(string path, string newpath) { ulong hash = new Jenkins96().ComputeHash(path); ulong newhash = new Jenkins96().ComputeHash(newpath); foreach (var root in Chunks) { if (!root.LocaleFlags.HasFlag(locale) && root != GlobalRoot) // ignore incorrect locale and not global { continue; } var entries = root.Entries.Where(x => x.NameHash == hash); foreach (var entry in entries) { var blte = CASContainer.EncodingHandler.CEKeys[entry.CEKey].EKeys[0]; entry.NameHash = newhash; entry.Path = path; CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, blte)); } } }
private void Worker_DoWork(object sender, DoWorkEventArgs e) { worker.ReportProgress(0, "Loading listfile.."); var linelist = new List <string>(); if (!File.Exists("listfile.txt")) { worker.ReportProgress(20, "Downloading listfile.."); UpdateListfile(); } if (!File.Exists("definitions/Map.dbd")) { worker.ReportProgress(30, "Downloading database definitions.."); UpdateDefinition("Map"); } worker.ReportProgress(50, "Loading listfile from disk.."); var hasher = new Jenkins96(); foreach (var line in File.ReadAllLines("listfile.txt")) { if (CASC.FileExists(line)) { linelist.Add(line.ToLower()); filenameLookup.Add(hasher.ComputeHash(line), line); } } worker.ReportProgress(70, "Sorting listfile.."); linelist.Sort(); var lines = linelist.ToArray(); linelist = null; var unwantedExtensions = new List <string>(); for (var u = 0; u < 512; u++) { unwantedExtensions.Add("_" + u.ToString().PadLeft(3, '0') + ".wmo"); } var unwanted = unwantedExtensions.ToArray(); unwantedExtensions = null; for (var i = 0; i < lines.Count(); i++) { if (showWMO && lines[i].EndsWith(".wmo")) { if (!unwanted.Contains(lines[i].Substring(lines[i].Length - 8, 8)) && !lines[i].EndsWith("lod.wmo") && !lines[i].EndsWith("lod1.wmo") && !lines[i].EndsWith("lod2.wmo") && !lines[i].EndsWith("lod3.wmo")) { models.Add(lines[i]); } } if (showM2 && lines[i].EndsWith(".m2")) { models.Add(lines[i]); } if (lines[i].EndsWith(".blp")) { textures.Add(lines[i]); } if (i % 1000 == 0) { var progress = (i * 100) / lines.Count(); worker.ReportProgress(progress, "Filtering listfile.."); } } lines = null; }
public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false) { this.minimumId = minimumid; this.locale = locale; string cdnPath = Helper.GetCDNPath("listfile.csv"); if (!(File.Exists(Path.Combine(CASContainer.Settings.OutputPath, cdnPath))) && onlineListfile) { CASContainer.Logger.LogInformation("Downloading listfile from WoW.Tools"); ListFileClient.DownloadFile("https://wow.tools/casc/listfile/download/csv/unverified", cdnPath); } BinaryReader stream = new BinaryReader(data); // 8.2 root change int magic = stream.ReadInt32(); bool newFormat = magic == headerMagic; if (newFormat) { allFiles = stream.ReadInt32(); namedFiles = stream.ReadInt32(); } else { stream.BaseStream.Position = 0; } long length = stream.BaseStream.Length; while (stream.BaseStream.Position < length) { RootChunk chunk = new RootChunk() { Count = stream.ReadUInt32(), ContentFlags = (ContentFlags)stream.ReadUInt32(), LocaleFlags = (LocaleFlags)stream.ReadUInt32(), }; parsedFiles += (int)chunk.Count; // set the global root if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None) { GlobalRoot = chunk; } uint fileDataIndex = 0; for (int i = 0; i < chunk.Count; i++) { uint offset = stream.ReadUInt32(); RootEntry entry = new RootEntry() { FileDataIdOffset = offset, FileDataId = fileDataIndex + offset }; fileDataIndex = entry.FileDataId + 1; chunk.Entries.Add(entry); } if (newFormat) { foreach (var entry in chunk.Entries) { entry.CEKey = new MD5Hash(stream); maxId = Math.Max(maxId, entry.FileDataId); } if (parsedFiles > allFiles - namedFiles) { foreach (var entry in chunk.Entries) { entry.NameHash = stream.ReadUInt64(); } } else // no namehash { foreach (var entry in chunk.Entries) { entry.NameHash = 0; } } } else { foreach (var entry in chunk.Entries) { entry.CEKey = new MD5Hash(stream); entry.NameHash = stream.ReadUInt64(); maxId = Math.Max(maxId, entry.FileDataId); } } Chunks.Add(chunk); } if (GlobalRoot == null) { CASContainer.Logger.LogCritical($"No Global root found. Root file is corrupt."); return; } // use listfile to assign names var listFileLines = File.ReadAllLines(cdnPath); foreach (var listFileData in listFileLines) { var splitData = listFileData.Split(';'); if (splitData.Length != 2) { continue; } if (!uint.TryParse(splitData[0], out uint listFileDataID)) { continue; } ListFile[listFileDataID] = new Jenkins96().ComputeHash(splitData[1]); } foreach (var chunk in Chunks) { foreach (var entry in chunk.Entries) { if (entry.NameHash == 0) { if (ListFile.ContainsKey(entry.FileDataId)) { entry.NameHash = ListFile[entry.FileDataId]; } } } } // set maxid from cache maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0); // store encoding map encodingMap = (data as BLTEStream)?.EncodingMap.FirstOrDefault() ?? new EncodingMap(EncodingType.ZLib, 9); stream?.Dispose(); data?.Dispose(); }
static bool CheckName(Jenkins96 hasher, string format, params object[] args) { string name = string.Format(format, args); return(CheckName(hasher, name)); }
static bool CheckName(Jenkins96 hasher, string name) { return(CheckName(hasher.ComputeHash(name), name)); }
public void Write() { foreach (var index in LocalIndices) { if (!index.Changed) { continue; } uint pC = 0; Jenkins96 hasher = new Jenkins96(); using (var ms = new MemoryStream()) using (var bw = new BinaryWriter(ms)) { bw.Write(index.HeaderHashSize); bw.Write((uint)0); // HeaderHash bw.Write(index._2); bw.Write(index.BucketIndex); bw.Write(index._4); bw.Write(index.EntrySizeBytes); bw.Write(index.EntryOffsetBytes); bw.Write(index.EntryKeyBytes); bw.Write(index.ArchiveFileHeaderBytes); bw.Write(index.ArchiveTotalSizeMaximum); bw.Write(new byte[8]); bw.Write((uint)index.Entries.Count * 18); bw.Write((uint)0); // EntriesHash // entries index.Entries.Sort(new HashComparer()); foreach (var entry in index.Entries) { bw.Write(entry.Key); bw.WriteUInt40BE(entry.ArchiveOffset); bw.Write(entry.Size); } // update EntriesHash bw.BaseStream.Position = 0x28; for (int i = 0; i < index.Entries.Count; i++) { byte[] entryhash = new byte[18]; bw.BaseStream.Read(entryhash, 0, entryhash.Length); hasher.ComputeHash(entryhash, out pC); } bw.BaseStream.Position = 0x24; bw.Write(pC); // update HeaderHash bw.BaseStream.Position = 8; byte[] headerhash = new byte[index.HeaderHashSize]; bw.BaseStream.Read(headerhash, 0, headerhash.Length); hasher.ComputeHash(headerhash, out pC, true); bw.BaseStream.Position = 4; bw.Write(pC); // minimum file length constraint if (bw.BaseStream.Length < CHUNK_SIZE) { bw.BaseStream.SetLength(CHUNK_SIZE); } // save file to output var bucket = index.BucketIndex.ToString("X2"); var version = long.Parse(Path.GetFileNameWithoutExtension(index.BaseFile).Substring(2), NumberStyles.HexNumber); string filename = bucket + version.ToString("X8") + ".idx"; var path = Path.Combine(CASContainer.Settings.OutputPath, Helper.GetCDNPath(filename.ToLowerInvariant(), "data")); using (var fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) { fs.Seek(0, SeekOrigin.End); ms.Position = 0; ms.CopyTo(fs); fs.Flush(); } index.Changed = false; } } }
static FileValidator() { Hashes = new HashSet <ulong>(); Filenames = new Queue <string>(); Jenkins96 = new Jenkins96(); }