static Jenkins96 HandleLine(string line, ParallelLoopState state, Jenkins96 hasher)
{
if (string.IsNullOrWhiteSpace(line))
{
return(hasher);
}
if (line.IndexOfAny(pathInvalidChars) != -1)
{
return(hasher);
}
line = line.Replace('/', '\\');
//string extLower = Path.GetExtension(line).ToLower();
//if (extLower == ".mdx" || extLower == ".mdl")
//{
// line = Path.ChangeExtension(line, ".m2");
// extLower = ".m2";
//}
ulong hash = hasher.ComputeHash(line);
if (!CheckName(hash, line))
{
return(hasher);
}
processed[hash] = true;
return(hasher);
}
static void Main(string[] args)
{
Console.WriteLine("Hashing devices found ::");
foreach (var AcceleratorDevice in AcceleratorDevice.All)
{
Console.WriteLine($" {AcceleratorDevice}");
}
Console.WriteLine("");
if (args == null || args.Length == 0)
{
throw new ArgumentException("No arguments supplied");
}
IHash hash;
switch (args[0].ToLowerInvariant())
{
case "salsa":
hash = new Salsa20();
break;
case "jenkins":
hash = new Jenkins96();
break;
case "benchmark":
BenchmarkJenkins(args);
return;
case "wordlist":
hash = new Wordlist();
break;
case "variation":
hash = new VarientGen();
break;
case "mix":
hash = new MixMatch();
break;
case "markov":
hash = new MarkovChain();
break;
default:
throw new ArgumentException("Invalid hash type");
}
// override console exit event
CleanExitHandler.Attach();
hash.LoadParameters(args);
hash.Start();
Console.ReadLine();
}
public void AddEntry(string path, CASResult file)
{
ulong hash = new Jenkins96().ComputeHash(path);
bool found = false;
// check to see if we're overwriting an existing entry
foreach (var root in Chunks)
{
if (!root.LocaleFlags.HasFlag(locale) && root != GlobalRoot) // skip incorrect locales and non-global roots
{
continue;
}
var index = root.Entries.FindIndex(x => x.Hash == hash);
if (index >= 0)
{
RootEntry entry = new RootEntry()
{
MD5 = file.DataHash,
Hash = hash,
Path = path,
FileDataId = root.Entries[index].FileDataId,
FileDataIdOffset = root.Entries[index].FileDataIdOffset
};
root.Entries[index] = entry; // update
found = true;
// max id check just to be safe
if (root == GlobalRoot)
{
maxId = Math.Max(entry.FileDataId, maxId); // update max id
}
CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash));
}
}
// must be a new file, add it to the global root
if (!found)
{
RootEntry entry = new RootEntry()
{
MD5 = file.DataHash,
FileDataIdOffset = 0,
Hash = hash,
Path = path
};
var cache = CASContainer.Settings.Cache.Entries.FirstOrDefault(x => x.Path == path);
if (cache?.Path != path) // get cache id
{
entry.FileDataId = Math.Max(maxId + 1, minimumId); // calculate the Id
}
GlobalRoot.Entries.Add(entry); // add new
maxId = Math.Max(entry.FileDataId, maxId); // Update max id
CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, file.Hash));
}
}
static void LocalFinal(Jenkins96 local)
{
int secs = (int)(DateTime.Now - startTime).TotalSeconds;
if (secs > 0)
{
currentSpeed = (int)(hashCount / secs);
}
ConsoleProgressBar.DrawProgressBar(currentGenerator.Percent, 71, '#', false, currentSpeed);
}
static void BenchmarkJenkins(params string[] args)
{
int perms = 5;
if (args != null && args.Length > 1 && !int.TryParse(args[1], out perms))
{
perms = 5;
}
perms = Math.Min(Math.Max(perms, 1), 9); // clamp to 1 - 9
char[] filename = "interface/cinematics/legion_dh2.mp3".ToCharArray(); // build the dummy template
for (int i = 0; i < perms; i++)
{
filename[30 - i] = '%';
}
Console.WriteLine($"Benchmarking Jenkins @ {perms} permutations (39^{perms}) :: ");
List <string> types = new List <string>()
{
"gpu", "cpu", "all"
}; // calculate the available devices
if (!AcceleratorDevice.HasCPU)
{
types.Remove("cpu");
}
if (!AcceleratorDevice.HasGPU)
{
types.Remove("gpu");
}
if (types.Count == 2 && AcceleratorDevice.All.Length == 1)
{
types.Remove("all");
}
Jenkins96 jenkins = new Jenkins96();
foreach (var type in types)
{
jenkins.LoadTestParameters(type, new string(filename));
Console.WriteLine("");
Console.WriteLine($"-- {type.ToUpper()} Test --");
jenkins.Start();
}
Console.ReadLine();
}
public void AddEntry(string path, CASResult file)
{
var cache = CASContainer.Settings.Cache;
ulong namehash = new Jenkins96().ComputeHash(path);
var entries = Chunks
.FindAll(chunk => chunk.LocaleFlags.HasFlag(locale)) // Select locales that match selected locale
.SelectMany(chunk => chunk.Entries) // Flatten the array to get all entries within all matching chunks
.Where(e => e.NameHash == namehash);
if (entries.Count() == 0)
{ // New file, we need to create an entry for it
var cached = cache.Entries.FirstOrDefault(x => x.Path == path);
var fileDataId = Math.Max(maxId + 1, minimumId);
if (cached != null)
{
fileDataId = cached.FileDataId;
}
var entry = new RootEntry()
{
CEKey = file.CEKey,
FileDataId = fileDataId,
FileDataIdOffset = 0,
NameHash = namehash,
Path = path
};
GlobalRoot.Entries.Add(entry); // Insert into the Global Root
maxId = Math.Max(entry.FileDataId, maxId); // Update the max id
cache?.AddOrUpdate(new CacheEntry(entry, file.EKey)); // If not done, sometimes files will not be added.
}
else
{ // Existing file, we just have to update the data hash
foreach (var entry in entries)
{
entry.CEKey = file.CEKey;
entry.Path = path;
cache?.AddOrUpdate(new CacheEntry(entry, file.EKey));
}
}
}
public void RemoveFile(string path)
{
ulong hash = new Jenkins96().ComputeHash(path);
foreach (var root in Chunks)
{
var entries = root.Entries.Where(x => x.NameHash == hash).ToArray(); // should only ever be one but just incase
foreach (var entry in entries)
{
if (CASContainer.EncodingHandler.CEKeys.TryGetValue(entry.CEKey, out EncodingCEKeyPageTable enc))
{
CASContainer.DownloadHandler?.RemoveEntry(enc.EKeys[0]); // remove from download
CASContainer.CDNIndexHandler?.RemoveEntry(enc.EKeys[0]); // remove from cdn index
}
root.Entries.Remove(entry);
CASContainer.Settings.Cache?.Remove(path);
}
}
}
static void Main(string[] args)
{
Console.WriteLine("Hashing devices found ::");
foreach (var AcceleratorDevice in AcceleratorDevice.All)
{
Console.WriteLine($" {AcceleratorDevice}");
}
Console.WriteLine("");
if (args == null || args.Length == 0)
{
throw new ArgumentException("No arguments supplied");
}
IHash hash;
switch (args[0].ToLowerInvariant())
{
case "salsa":
hash = new Salsa20();
break;
case "jenkins":
hash = new Jenkins96();
break;
case "benchmark":
BenchmarkJenkins(args);
return;
default:
throw new ArgumentException("Invalid hash type");
}
hash.LoadParameters(args);
hash.Start();
}
public void RenameFile(string path, string newpath)
{
ulong hash = new Jenkins96().ComputeHash(path);
ulong newhash = new Jenkins96().ComputeHash(newpath);
foreach (var root in Chunks)
{
if (!root.LocaleFlags.HasFlag(locale) && root != GlobalRoot) // ignore incorrect locale and not global
{
continue;
}
var entries = root.Entries.Where(x => x.NameHash == hash);
foreach (var entry in entries)
{
var blte = CASContainer.EncodingHandler.CEKeys[entry.CEKey].EKeys[0];
entry.NameHash = newhash;
entry.Path = path;
CASContainer.Settings.Cache?.AddOrUpdate(new CacheEntry(entry, blte));
}
}
}
private void Worker_DoWork(object sender, DoWorkEventArgs e)
{
worker.ReportProgress(0, "Loading listfile..");
var linelist = new List <string>();
if (!File.Exists("listfile.txt"))
{
worker.ReportProgress(20, "Downloading listfile..");
UpdateListfile();
}
if (!File.Exists("definitions/Map.dbd"))
{
worker.ReportProgress(30, "Downloading database definitions..");
UpdateDefinition("Map");
}
worker.ReportProgress(50, "Loading listfile from disk..");
var hasher = new Jenkins96();
foreach (var line in File.ReadAllLines("listfile.txt"))
{
if (CASC.FileExists(line))
{
linelist.Add(line.ToLower());
filenameLookup.Add(hasher.ComputeHash(line), line);
}
}
worker.ReportProgress(70, "Sorting listfile..");
linelist.Sort();
var lines = linelist.ToArray();
linelist = null;
var unwantedExtensions = new List <string>();
for (var u = 0; u < 512; u++)
{
unwantedExtensions.Add("_" + u.ToString().PadLeft(3, '0') + ".wmo");
}
var unwanted = unwantedExtensions.ToArray();
unwantedExtensions = null;
for (var i = 0; i < lines.Count(); i++)
{
if (showWMO && lines[i].EndsWith(".wmo"))
{
if (!unwanted.Contains(lines[i].Substring(lines[i].Length - 8, 8)) && !lines[i].EndsWith("lod.wmo") && !lines[i].EndsWith("lod1.wmo") && !lines[i].EndsWith("lod2.wmo") && !lines[i].EndsWith("lod3.wmo"))
{
models.Add(lines[i]);
}
}
if (showM2 && lines[i].EndsWith(".m2"))
{
models.Add(lines[i]);
}
if (lines[i].EndsWith(".blp"))
{
textures.Add(lines[i]);
}
if (i % 1000 == 0)
{
var progress = (i * 100) / lines.Count();
worker.ReportProgress(progress, "Filtering listfile..");
}
}
lines = null;
}
public RootHandler(Stream data, LocaleFlags locale, uint minimumid = 0, bool onlineListfile = false)
{
this.minimumId = minimumid;
this.locale = locale;
string cdnPath = Helper.GetCDNPath("listfile.csv");
if (!(File.Exists(Path.Combine(CASContainer.Settings.OutputPath, cdnPath))) && onlineListfile)
{
CASContainer.Logger.LogInformation("Downloading listfile from WoW.Tools");
ListFileClient.DownloadFile("https://wow.tools/casc/listfile/download/csv/unverified", cdnPath);
}
BinaryReader stream = new BinaryReader(data);
// 8.2 root change
int magic = stream.ReadInt32();
bool newFormat = magic == headerMagic;
if (newFormat)
{
allFiles = stream.ReadInt32();
namedFiles = stream.ReadInt32();
}
else
{
stream.BaseStream.Position = 0;
}
long length = stream.BaseStream.Length;
while (stream.BaseStream.Position < length)
{
RootChunk chunk = new RootChunk()
{
Count = stream.ReadUInt32(),
ContentFlags = (ContentFlags)stream.ReadUInt32(),
LocaleFlags = (LocaleFlags)stream.ReadUInt32(),
};
parsedFiles += (int)chunk.Count;
// set the global root
if (chunk.LocaleFlags == LocaleFlags.All_WoW && chunk.ContentFlags == ContentFlags.None)
{
GlobalRoot = chunk;
}
uint fileDataIndex = 0;
for (int i = 0; i < chunk.Count; i++)
{
uint offset = stream.ReadUInt32();
RootEntry entry = new RootEntry()
{
FileDataIdOffset = offset,
FileDataId = fileDataIndex + offset
};
fileDataIndex = entry.FileDataId + 1;
chunk.Entries.Add(entry);
}
if (newFormat)
{
foreach (var entry in chunk.Entries)
{
entry.CEKey = new MD5Hash(stream);
maxId = Math.Max(maxId, entry.FileDataId);
}
if (parsedFiles > allFiles - namedFiles)
{
foreach (var entry in chunk.Entries)
{
entry.NameHash = stream.ReadUInt64();
}
}
else // no namehash
{
foreach (var entry in chunk.Entries)
{
entry.NameHash = 0;
}
}
}
else
{
foreach (var entry in chunk.Entries)
{
entry.CEKey = new MD5Hash(stream);
entry.NameHash = stream.ReadUInt64();
maxId = Math.Max(maxId, entry.FileDataId);
}
}
Chunks.Add(chunk);
}
if (GlobalRoot == null)
{
CASContainer.Logger.LogCritical($"No Global root found. Root file is corrupt.");
return;
}
// use listfile to assign names
var listFileLines = File.ReadAllLines(cdnPath);
foreach (var listFileData in listFileLines)
{
var splitData = listFileData.Split(';');
if (splitData.Length != 2)
{
continue;
}
if (!uint.TryParse(splitData[0], out uint listFileDataID))
{
continue;
}
ListFile[listFileDataID] = new Jenkins96().ComputeHash(splitData[1]);
}
foreach (var chunk in Chunks)
{
foreach (var entry in chunk.Entries)
{
if (entry.NameHash == 0)
{
if (ListFile.ContainsKey(entry.FileDataId))
{
entry.NameHash = ListFile[entry.FileDataId];
}
}
}
}
// set maxid from cache
maxId = Math.Max(Math.Max(maxId, minimumid), CASContainer.Settings.Cache?.MaxId ?? 0);
// store encoding map
encodingMap = (data as BLTEStream)?.EncodingMap.FirstOrDefault() ?? new EncodingMap(EncodingType.ZLib, 9);
stream?.Dispose();
data?.Dispose();
}
static bool CheckName(Jenkins96 hasher, string format, params object[] args)
{
string name = string.Format(format, args);
return(CheckName(hasher, name));
}
static bool CheckName(Jenkins96 hasher, string name)
{
return(CheckName(hasher.ComputeHash(name), name));
}
public void Write()
{
foreach (var index in LocalIndices)
{
if (!index.Changed)
{
continue;
}
uint pC = 0;
Jenkins96 hasher = new Jenkins96();
using (var ms = new MemoryStream())
using (var bw = new BinaryWriter(ms))
{
bw.Write(index.HeaderHashSize);
bw.Write((uint)0); // HeaderHash
bw.Write(index._2);
bw.Write(index.BucketIndex);
bw.Write(index._4);
bw.Write(index.EntrySizeBytes);
bw.Write(index.EntryOffsetBytes);
bw.Write(index.EntryKeyBytes);
bw.Write(index.ArchiveFileHeaderBytes);
bw.Write(index.ArchiveTotalSizeMaximum);
bw.Write(new byte[8]);
bw.Write((uint)index.Entries.Count * 18);
bw.Write((uint)0); // EntriesHash
// entries
index.Entries.Sort(new HashComparer());
foreach (var entry in index.Entries)
{
bw.Write(entry.Key);
bw.WriteUInt40BE(entry.ArchiveOffset);
bw.Write(entry.Size);
}
// update EntriesHash
bw.BaseStream.Position = 0x28;
for (int i = 0; i < index.Entries.Count; i++)
{
byte[] entryhash = new byte[18];
bw.BaseStream.Read(entryhash, 0, entryhash.Length);
hasher.ComputeHash(entryhash, out pC);
}
bw.BaseStream.Position = 0x24;
bw.Write(pC);
// update HeaderHash
bw.BaseStream.Position = 8;
byte[] headerhash = new byte[index.HeaderHashSize];
bw.BaseStream.Read(headerhash, 0, headerhash.Length);
hasher.ComputeHash(headerhash, out pC, true);
bw.BaseStream.Position = 4;
bw.Write(pC);
// minimum file length constraint
if (bw.BaseStream.Length < CHUNK_SIZE)
{
bw.BaseStream.SetLength(CHUNK_SIZE);
}
// save file to output
var bucket = index.BucketIndex.ToString("X2");
var version = long.Parse(Path.GetFileNameWithoutExtension(index.BaseFile).Substring(2), NumberStyles.HexNumber);
string filename = bucket + version.ToString("X8") + ".idx";
var path = Path.Combine(CASContainer.Settings.OutputPath, Helper.GetCDNPath(filename.ToLowerInvariant(), "data"));
using (var fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
fs.Seek(0, SeekOrigin.End);
ms.Position = 0;
ms.CopyTo(fs);
fs.Flush();
}
index.Changed = false;
}
}
}
static FileValidator()
{
Hashes = new HashSet <ulong>();
Filenames = new Queue <string>();
Jenkins96 = new Jenkins96();
}
