/// <summary>
/// Enumerates all non-archive local files, BLT encodes them and adds them to the InstallFile
/// </summary>
/// <param name="maxDegreeOfParallelism"></param>
/// <returns></returns>
public void ExportFiles(int maxDegreeOfParallelism = 15)
{
var results = new ConcurrentBag <CASRecord>();
var block = new ActionBlock <string>(file =>
{
// strip the local path and normalise
var name = file[(file.IndexOf(BaseDirectory, Comparison) + BaseDirectory.Length)..].WoWNormalise();
// block table encode and export to the temp folder
// then add appropiate tags
var record = BlockTableEncoder.EncodeAndExport(file, Options.TempDirectory, name);
record.Tags = TagGenerator.GetTags(file);
if (!EncodingCache.ContainsEKey(record.EKey))
{
EncodingCache.AddOrUpdate(record);
}
else
{
record.BLTEPath = "";
}
results.Add(record);
},
/// <summary>
/// Extracts a collection of files from an archive and BLTE encodes them
/// </summary>
/// <param name="mpq"></param>
/// <param name="filenames"></param>
/// <param name="maxDegreeOfParallelism"></param>
/// <returns></returns>
private async Task ExportFiles(MpqArchive mpq, IEnumerable <string> filenames, bool applyTags = false, int maxDegreeOfParallelism = 150)
{
var block = new ActionBlock <string>(file =>
{
using var fs = mpq.OpenFile(file);
// ignore PTCH files
if (fs.Flags.HasFlag(MPQFileAttributes.PatchFile))
{
return;
}
// patch has marked file for deletion so remove from filelist
if (fs.Flags.HasFlag(MPQFileAttributes.DeleteMarker))
{
FileList.TryRemove(file, out _);
return;
}
if (fs.CanRead && fs.Length > 0)
{
var map = BlockTableEncoder.GetEMapFromExtension(file, fs.Length);
if (!EncodingCache.TryGetRecord(MD5Hash.Parse(fs.GetMD5Hash()), file, out var record))
{
record = BlockTableEncoder.EncodeAndExport(fs, map, Options.TempDirectory, file);
EncodingCache.AddOrUpdate(record);
}
if (applyTags)
{
record.Tags = TagGenerator.GetTags(file, fs);
}
record.EBlock.EncodingMap = map;
FileList.TryAdd(file, record);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
foreach (var file in filenames)
{
if (!FileList.ContainsKey(file))
{
block.Post(file);
}
}
block.Complete();
await block.Completion;
}
private async Task ExtractFiles(UnshippedBuild model, EncodingFile encoding)
{
var install = new InstallFile(Helpers.GetTempPath(model.Install));
foreach (var file in install.Files)
{
if (!Helpers.HasAnyWildcard(file.FilePath, Consts.FileWildcards))
{
continue;
}
if (!encoding.TryGetCKeyEntry(file.CKey, out var contentEntry))
{
continue;
}
if (contentEntry.EKeys.Count == 0)
{
continue;
}
var filepath = Path.Combine(Consts.TempDir, file.FilePath);
Directory.CreateDirectory(Path.GetDirectoryName(filepath));
// download the first one
foreach (var ekey in contentEntry.EKeys)
{
if (await DownloadFile(ekey.ToString(), filepath))
{
try
{
// decode file, using swap temp to avoid rw conflicts
BlockTableEncoder.DecodeAndExport(filepath, filepath + "_temp");
File.Copy(filepath + "_temp", filepath, true);
}
finally
{
File.Delete(filepath + "_temp");
}
// extract build info
if (file.FilePath.EndsWith(".exe") && File.Exists(filepath))
{
model.Build ??= FileVersionInfo.GetVersionInfo(filepath)?.FileVersion;
model.Product ??= Helpers.GetProductType(file.FilePath);
}
break;
}
}
}
}
/// <summary>
/// Iterates all loose files within the data directory and BLT encodes them
/// </summary>
/// <param name="filenames"></param>
public void EnumerateLooseDataFiles(IEnumerable <string> filenames)
{
if (!filenames.Any())
{
return;
}
Log.WriteLine("Exporting loose Data files");
var block = new ActionBlock <string>(file =>
{
var filename = GetInternalPath(file);
var record = BlockTableEncoder.EncodeAndExport(file, Options.TempDirectory, filename);
record.Tags = TagGenerator.GetTags(file);
if (!EncodingCache.ContainsEKey(record.EKey))
{
EncodingCache.AddOrUpdate(record);
}
else
{
record.BLTEPath = "";
}
FileList.TryAdd(filename, record);
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 150
});
foreach (var f in filenames)
{
block.Post(f);
}
block.Complete();
block.Completion.Wait();
}
/// <summary>
/// Some alpha MPQs are hotswappable and only contain a single file and it's checksum
/// </summary>
/// <param name="mpq"></param>
/// <param name="archivename"></param>
private bool TryReadAlpha(MpqArchive mpq, string archivename)
{
// strip the local path and extension to get the filename
var file = Path.ChangeExtension(GetInternalPath(archivename), null).WoWNormalise();
if (FileList.ContainsKey(file))
{
return(true);
}
// add the filename as the listfile
var internalname = Path.GetFileName(file);
mpq.AddListFile(internalname);
// read file if known
if (mpq.HasFile(internalname))
{
using var fs = mpq.OpenFile(internalname);
if (fs.CanRead && fs.Length > 0)
{
var map = BlockTableEncoder.GetEMapFromExtension(file, fs.Length);
if (!EncodingCache.TryGetRecord(MD5Hash.Parse(fs.GetMD5Hash()), file, out var record))
{
record = BlockTableEncoder.EncodeAndExport(fs, map, Options.TempDirectory, file);
EncodingCache.AddOrUpdate(record);
}
record.EBlock.EncodingMap = map;
FileList.TryAdd(file, record);
return(true);
}
}
return(false);
}
public static string MANIFEST_PATH; // = CdnRootDir + @"\wow";
static void Main(string[] args)
{
string seetingsDir = Directory.GetCurrentDirectory() + "/AppSettings.json";
AppSettings settings = null;
try
{
FileStream f = new FileStream(seetingsDir, FileMode.Open, FileAccess.Read, FileShare.Read);
var streamReader = new StreamReader(f, Encoding.UTF8);
settings = JsonConvert.DeserializeObject <AppSettings>(streamReader.ReadToEnd());
}
catch (Exception e)
{
settings = new AppSettings
{
cdnRootDir = "",
mysqlUser = "******",
mysqlPassword = "******",
dataFolderPath = Directory.GetCurrentDirectory() + "\\Data\\",
exportFolderPath = Directory.GetCurrentDirectory() + "\\DataExport\\",
databaseName = "tacthost",
databaseAdress = "localhost"
};
string json = JsonConvert.SerializeObject(settings, Formatting.Indented);
File.WriteAllText(seetingsDir, json);
}
CDN_DATA_DIR = settings.cdnRootDir + @"tpr\wow";
MANIFEST_PATH = settings.cdnRootDir + @"wow";
var tactRepo = new TACTRepo(CDN_DATA_DIR)
{
ManifestContainer = new ManifestContainer("wow", Locale.EU),
ConfigContainer = new ConfigContainer()
};
tactRepo.ManifestContainer.OpenLocal(MANIFEST_PATH);
tactRepo.ConfigContainer.OpenLocal(tactRepo.BaseDirectory, tactRepo.ManifestContainer);
tactRepo.IndexContainer = new Net.Indices.IndexContainer();
tactRepo.IndexContainer.Open(tactRepo.BaseDirectory);
tactRepo.EncodingFile = new Net.Encoding.EncodingFile(tactRepo.BaseDirectory, tactRepo.ConfigContainer.EncodingEKey);
tactRepo.EncodingFile.TryGetCKeyEntry(tactRepo.ConfigContainer.RootCKey, out var rootCEntry);
tactRepo.RootFile = new Net.Root.RootFile(tactRepo.BaseDirectory, rootCEntry.EKey);
tactRepo.InstallFile = new InstallFile(tactRepo.BaseDirectory, tactRepo.ConfigContainer.InstallEKey);
tactRepo.DownloadFile = new Net.Download.DownloadFile(tactRepo.BaseDirectory, tactRepo.ConfigContainer.DownloadEKey);
tactRepo.PatchFile = new Net.Patch.PatchFile(tactRepo.BaseDirectory, tactRepo.ConfigContainer.PatchEKey);
tactRepo.RootFile.FileLookup = new ListFileLookup();
string[] files = Directory.GetFiles(settings.dataFolderPath, "*", SearchOption.AllDirectories);
for (int i = 0; i < files.Length; i++)
{
files[i] = files[i].Replace(settings.dataFolderPath, "");
}
Console.WriteLine("Found " + files.Length + " Files to proccess in " + settings.dataFolderPath);
foreach (string s in files)
{
string filename = s.Substring(s.LastIndexOf("\\") + 1);
string path = s.Substring(0, s.LastIndexOf("\\") + 1);
LocaleFlags locale = LocaleFlags.All_WoW;
if (path.ToLower().Contains("dbfilesclient"))
{
var match = Regex.Match(path, @"(frFR)|(enUS)|(ptBR)|(ruRU)|(esES)|(esMX)|(itIT)|(deDE)|(enGB)|(ptPT)");
var pathSplit = path.Split("\\");
var localeString = "";
foreach (var split in pathSplit)
{
if (split.Contains(match.Value))
{
locale = (LocaleFlags)Enum.Parse(typeof(LocaleFlags), split);
localeString = split;
path = path.Replace("\\" + localeString, "");
break;
}
}
}
CASRecord record = BlockTableEncoder.EncodeAndExport(settings.dataFolderPath + s, settings.exportFolderPath, (path + filename).Replace("\\", "/"));
uint fId = tactRepo.RootFile.FileLookup.GetOrCreateFileId(record.FileName);
Console.WriteLine("Added " + filename + " with ID: " + fId);
tactRepo.RootFile.AddOrUpdate(record, tactRepo, locale);
tactRepo.InstallFile.AddOrUpdate(record, tactRepo);
}
tactRepo.Save(tactRepo.BaseDirectory, settings.cdnRootDir);
}
