public MapFile Read(DatFile dataFileReference = null) { List <IGenericFieldBlock> fields = new List <IGenericFieldBlock>(); using (FileStream fs = new FileStream(this.infile, FileMode.Open, FileAccess.Read)) { using (BinaryReader r = new BinaryReader(fs)) { // construct an object using the file spec char[] fileID = r.ReadChars(4); if (string.Compare(new string(fileID), new string(MAGIC)) != 0) { throw new ArgumentException("Not a valid Army Men 1 map file"); } // checknum UInt32 fileContentSizeBytes = r.ReadUInt32(); // not including magic id and file size _ = r.ReadChars(4); // "MAP " identifier // I dont know that this is correct way to read the version int major = 6; int minor = 0; while (r.BaseStream.Position != r.BaseStream.Length) { char[] fieldID = r.ReadChars(4); if (r.BaseStream.Position == r.BaseStream.Length) // some files appear to have an extra nil at the end { break; } // there seems to be a bug in the OATT field length int, so we need to parse it bit by bit string id = new string(fieldID); Int32 size; switch (id) { case "TLAY": TNAMBlock tNAMBlock = (TNAMBlock)fields.FindLast(x => x.DisplayFieldName == "Textures"); fields.Add(new TLAYBlock(r, tNAMBlock)); break; case "TNAM": fields.Add(new TNAMBlock(r, Directory.GetParent(this.infile).FullName)); break; case "OLAY": fields.Add(new OLAYBlock(r)); break; case "OATT": OLAYBlock lastOLAY = (OLAYBlock)fields.FindLast(x => x.DisplayFieldName == OLAYBlock.FIELD_NAME); fields.Add(new OATTBlock(r, lastOLAY)); break; case "SCEN": fields.Add(new SCENBlock(r)); break; case "VERS": size = r.ReadInt32(); // length of version field. Always 8. // this field always occurs either first or after CNUM. CNUM doesnt always appear. major = r.ReadInt32(); minor = r.ReadInt32(); Span <byte> content = stackalloc byte[size]; BinaryPrimitives.WriteInt32LittleEndian(content.Slice(0, 4), major); BinaryPrimitives.WriteInt32LittleEndian(content.Slice(4, 4), minor); fields.Add(new GenericFieldBlock(fieldID, size, content.ToArray())); break; default: size = r.ReadInt32(); { byte[] c = r.ReadBytes(size); fields.Add(new GenericFieldBlock(fieldID, size, c)); } break; } } } } fields.Add( new CombinedTextureLayersBlock( fields.OfType <TNAMBlock>().First(), fields .OfType <TLAYBlock>() .ToList(), fields.OfType <OLAYBlock>().ToList(), dataFileReference ) ); return(new MapFile(fields)); }
/// <summary> /// Wrap adding files to the depots /// </summary> /// <param name="inputs">List of input folders to use</param> /// <param name="onlyNeeded">True if only files in the database and don't exist are added, false otherwise</param> /// <param name="resume">Resume a previously interrupted operation from the specified path</param> /// <param name="includeZips">flag value == 0 means: add Zip files themselves into the depot in addition to their contents, flag value == 2 means add Zip files themselves but don't add content</param> /// <param name="workers">How many workers to launch for the job, default from config</param> /// <param name="includeGZips">flag value == 0 means: add GZip files themselves into the depot in addition to their contents, flag value == 2 means add GZip files themselves but don't add content</param> /// <param name="include7Zips">flag value == 0 means: add 7Zip files themselves into the depot in addition to their contents, flag value == 2 means add 7Zip files themselves but don't add content</param> /// <param name="skipInitialScan">True to skip the initial scan of the files to determine amount of work, false otherwise</param> /// <param name="useGolangZip">True to use go zip implementation instead of zlib, false otherwise</param> /// <param name="noDb">True to archive into depot but do not touch DB index and ignore only-needed flag, false otherwise</param> /// TODO: Add ability to update .romba files with proper size AND use the correct depot if it fills up /// TODO: Add ability correctly to mark which depot the files are being rebuilt to in the DB private static void InitArchive( List <string> inputs, bool onlyNeeded, string resume, int includeZips, int workers, int includeGZips, int include7Zips, bool skipInitialScan, bool useGolangZip, // Obsolete bool noDb) { // First we want to get just all directories from the inputs List <string> onlyDirs = new List <string>(); foreach (string input in inputs) { if (Directory.Exists(input)) { onlyDirs.Add(Path.GetFullPath(input)); } } // Then process all of the input directories into an internal DAT DatFile df = new DatFile(); foreach (string dir in onlyDirs) { // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually df.PopulateFromDir(dir, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true); df.PopulateFromDir(dir, Hash.DeepHashes, false, true, SkipFileType.None, false, false, _tmpdir, false, null, true); } // Create an empty Dat for files that need to be rebuilt DatFile need = new DatFile(); // Open the database connection SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); // Now that we have the Dats, add the files to the database string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES"; string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; foreach (string key in df.Keys) { List <DatItem> datItems = df[key]; foreach (Rom rom in datItems) { // If we care about if the file exists, check the databse first if (onlyNeeded && !noDb) { string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1" + " WHERE crcsha1.crc=\"" + rom.CRC + "\"" + " OR md5sha1.md5=\"" + rom.MD5 + "\"" + " OR md5sha1.sha1=\"" + rom.SHA1 + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) { // Add to the queries if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcquery += " (\"" + rom.CRC + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5query += " (\"" + rom.MD5 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += " (\"" + rom.SHA1 + "\", \"" + _depots.Keys.ToList()[0] + "\"),"; if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),"; } } // Add to the Dat need.Add(key, rom); } } // Otherwise, just add the file to the list else { // Add to the queries if (!noDb) { if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcquery += " (\"" + rom.CRC + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5query += " (\"" + rom.MD5 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += " (\"" + rom.SHA1 + "\", \"" + _depots.Keys.ToList()[0] + "\"),"; if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),"; } } } // Add to the Dat need.Add(key, rom); } } } // Now run the queries, if they're populated if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") { SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") { SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES") { SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") { SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") { SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(include7Zips, includeGZips, 2, includeZips); need.RebuildGeneric(onlyDirs, _depots.Keys.ToList()[0], false /*quickScan*/, false /*date*/, false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzip, true /*romba*/, asl, false /*updateDat*/, null /*headerToCheckAgainst*/, true /* chdsAsFiles */); }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> public ArchiveDotOrg(DatFile datFile) : base(datFile) { }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> public AttractMode(DatFile datFile) : base(datFile) { }
/// <summary> /// Process the DAT and find all matches in input files and folders /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="inputs">List of input files/folders to check</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>True if rebuilding was a success, false otherwise</returns> public static bool RebuildGeneric( DatFile datFile, List <string> inputs, string outDir, bool quickScan = false, bool date = false, bool delete = false, bool inverse = false, OutputFormat outputFormat = OutputFormat.Folder, TreatAsFile asFiles = 0x00) { #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit if (datFile.Items.TotalCount == 0 && !inverse) { logger.User("No entries were found to rebuild, exiting..."); return(false); } // Check that the output directory exists if (!Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); outDir = Path.GetFullPath(outDir); } // Now we want to get forcepack flag if it's not overridden if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None) { outputFormat = GetOutputFormat(datFile.Header.ForcePacking); } #endregion bool success = true; #region Rebuild from sources in order string format = FromOutputFormat(outputFormat) ?? string.Empty; InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}"); // Now loop through all of the files in all of the inputs foreach (string input in inputs) { // If the input is a file if (File.Exists(input)) { logger.User($"Checking file: {input}"); bool rebuilt = RebuildGenericHelper(datFile, input, outDir, quickScan, date, inverse, outputFormat, asFiles); // If we are supposed to delete the file, do so if (delete && rebuilt) { File.Delete(input); } } // If the input is a directory else if (Directory.Exists(input)) { logger.Verbose($"Checking directory: {input}"); foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) { logger.User($"Checking file: {file}"); bool rebuilt = RebuildGenericHelper(datFile, file, outDir, quickScan, date, inverse, outputFormat, asFiles); // If we are supposed to delete the file, do so if (delete && rebuilt) { File.Delete(input); } } } } watch.Stop(); #endregion return(success); }
/// <summary> /// Find duplicates and rebuild individual files to output /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="datItem">Information for the current file to rebuild from</param> /// <param name="file">Name of the file to process</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param> /// <returns>True if the file was able to be rebuilt, false otherwise</returns> private static bool RebuildIndividualFile( DatFile datFile, DatItem datItem, string file, string outDir, bool date, bool inverse, OutputFormat outputFormat, bool?isZip = null) { // Set the initial output value bool rebuilt = false; // If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) && !(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) && !(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba)) { outputFormat = OutputFormat.Folder; } // If we have a Disk or Media, change it into a Rom for later use if (datItem.ItemType == ItemType.Disk) { datItem = (datItem as Disk).ConvertToRom(); } else if (datItem.ItemType == ItemType.Media) { datItem = (datItem as Media).ConvertToRom(); } // Prepopluate a key string string crc = (datItem as Rom).CRC ?? string.Empty; // Try to get the stream for the file if (!GetFileStream(datItem, file, isZip, out Stream fileStream)) { return(false); } // If either we have duplicates or we're filtering if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes)) { // If we have a very specific TGZ->TGZ case, just copy it accordingly if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip)) { return(true); } // If we have a very specific TXZ->TXZ case, just copy it accordingly if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip)) { return(true); } logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly..."); rebuilt = true; // Special case for partial packing mode bool shouldCheck = false; if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial) { shouldCheck = true; datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false); } // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { // If we should check for the items in the machine if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1) { outputFormat = OutputFormat.Folder; } else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1) { outputFormat = OutputFormat.ParentFolder; } // Get the output archive, if possible Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat); // Now rebuild to the output file outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile()); } // Close the input stream fileStream?.Dispose(); } // Now we want to take care of headers, if applicable if (datFile.Header.HeaderSkipper != null) { // Check to see if we have a matching header first SkipperMatch.Init(); SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper)); // If there's a match, create the new file to write if (rule.Tests != null && rule.Tests.Count != 0) { // If the file could be transformed correctly MemoryStream transformStream = new MemoryStream(); if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true)) { // Get the file informations that we will be using Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true)); // If we have duplicates and we're not filtering if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes)) { logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly..."); rebuilt = true; // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { // Create a headered item to use as well datItem.CopyMachineInformation(item); datItem.SetName($"{datItem.GetName()}_{crc}"); // Get the output archive, if possible Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat); // Now rebuild to the output file bool eitherSuccess = false; eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile()); eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile()); // Now add the success of either rebuild rebuilt &= eitherSuccess; } } } // Dispose of the stream transformStream?.Dispose(); } // Dispose of the stream fileStream?.Dispose(); } return(rebuilt); }
/// <summary> /// Wrap converting and updating DAT file from any format to any format /// </summary> /// <param name="inputPaths">List of input filenames</param> /// <param name="basePaths">List of base filenames</param> /// /* Normal DAT header info */ /// <param name="datHeader">All DatHeader info to be used</param> /// /* Merging and Diffing info */ /// <param name="updateMode">Non-zero flag for diffing mode, zero otherwise</param> /// <param name="inplace">True if the cascade-diffed files should overwrite their inputs, false otherwise</param> /// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param> /// <param name="bare">True if the date should not be appended to the default name, false otherwise</param> /// /* Filtering info */ /// <param name="filter">Pre-populated filter object for DAT filtering</param> /// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param> /// /* Output DAT info */ /// <param name="outDir">Optional param for output directory</param> /// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param> /// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param> /// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param> /// <param name="replaceMode">ReplaceMode representing what should be updated [only for base replacement]</param> /// <param name="onlySame">True if descriptions should only be replaced if the game name is the same, false otherwise [only for base replacement]</param> private static void InitUpdate( List <string> inputPaths, List <string> basePaths, /* Normal DAT header info */ DatHeader datHeader, /* Merging and Diffing info */ UpdateMode updateMode, bool inplace, bool skip, bool bare, /* Filtering info */ Filter filter, SplitType splitType, /* Output DAT info */ string outDir, bool clean, bool remUnicode, bool descAsName, ReplaceMode replaceMode, bool onlySame) { // Normalize the extensions datHeader.AddExtension = (String.IsNullOrWhiteSpace(datHeader.AddExtension) || datHeader.AddExtension.StartsWith(".") ? datHeader.AddExtension : "." + datHeader.AddExtension); datHeader.ReplaceExtension = (String.IsNullOrWhiteSpace(datHeader.ReplaceExtension) || datHeader.ReplaceExtension.StartsWith(".") ? datHeader.ReplaceExtension : "." + datHeader.ReplaceExtension); // If we're in a special update mode and the names aren't set, set defaults if (updateMode != 0) { // Get the values that will be used if (String.IsNullOrWhiteSpace(datHeader.Date)) { datHeader.Date = DateTime.Now.ToString("yyyy-MM-dd"); } if (String.IsNullOrWhiteSpace(datHeader.Name)) { datHeader.Name = (updateMode != 0 ? "DiffDAT" : "MergeDAT") + (datHeader.Type == "SuperDAT" ? "-SuperDAT" : "") + (datHeader.DedupeRoms != DedupeType.None ? "-deduped" : ""); } if (String.IsNullOrWhiteSpace(datHeader.Description)) { datHeader.Description = (updateMode != 0 ? "DiffDAT" : "MergeDAT") + (datHeader.Type == "SuperDAT" ? "-SuperDAT" : "") + (datHeader.DedupeRoms != DedupeType.None ? " - deduped" : ""); if (!bare) { datHeader.Description += " (" + datHeader.Date + ")"; } } if (String.IsNullOrWhiteSpace(datHeader.Category) && updateMode != 0) { datHeader.Category = "DiffDAT"; } if (String.IsNullOrWhiteSpace(datHeader.Author)) { datHeader.Author = "SabreTools"; } } // If no replacement mode is set, default to Names if (replaceMode == ReplaceMode.None) { replaceMode = ReplaceMode.ItemName; } // Populate the DatData object DatFile userInputDat = new DatFile(datHeader); userInputDat.DetermineUpdateType(inputPaths, basePaths, outDir, updateMode, inplace, skip, clean, remUnicode, descAsName, filter, splitType, replaceMode, onlySame); }
public override bool ProcessFeatures(Dictionary <string, Feature> features) { // If the base fails, just fail out if (!base.ProcessFeatures(features)) { return(false); } // Get feature flags TreatAsFile asFiles = GetTreatAsFiles(features); bool date = GetBoolean(features, AddDateValue); bool delete = GetBoolean(features, DeleteValue); bool inverse = GetBoolean(features, InverseValue); bool quickScan = GetBoolean(features, QuickValue); bool updateDat = GetBoolean(features, UpdateDatValue); var outputFormat = GetOutputFormat(features); // If we have the romba flag if (Header.OutputDepot?.IsActive == true) { // Update TorrentGzip output if (outputFormat == OutputFormat.TorrentGzip) { outputFormat = OutputFormat.TorrentGzipRomba; } // Update TorrentXz output else if (outputFormat == OutputFormat.TorrentXZ) { outputFormat = OutputFormat.TorrentXZRomba; } } // Get a list of files from the input datfiles var datfiles = GetList(features, DatListValue); var datfilePaths = PathTool.GetFilesOnly(datfiles); // If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir if (GetBoolean(features, IndividualValue)) { foreach (ParentablePath datfile in datfilePaths) { DatFile datdata = DatFile.Create(); Parser.ParseInto(datdata, datfile, int.MaxValue, keep: true); // Set depot information datdata.Header.InputDepot = Header.InputDepot?.Clone() as DepotInformation; datdata.Header.OutputDepot = Header.OutputDepot?.Clone() as DepotInformation; // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) { datdata.Header.HeaderSkipper = Header.HeaderSkipper; } // If we have the depot flag, respect it bool success; if (Header.InputDepot?.IsActive ?? false) { success = Rebuilder.RebuildDepot(datdata, Inputs, Path.Combine(OutputDir, datdata.Header.FileName), date, delete, inverse, outputFormat); } else { success = Rebuilder.RebuildGeneric(datdata, Inputs, Path.Combine(OutputDir, datdata.Header.FileName), quickScan, date, delete, inverse, outputFormat, asFiles); } // If we have a success and we're updating the DAT, write it out if (success && updateDat) { datdata.Header.FileName = $"fixDAT_{Header.FileName}"; datdata.Header.Name = $"fixDAT_{Header.Name}"; datdata.Header.Description = $"fixDAT_{Header.Description}"; datdata.Items.ClearMarked(); Writer.Write(datdata, OutputDir); } } } // Otherwise, process all DATs into the same output else { InternalStopwatch watch = new InternalStopwatch("Populating internal DAT"); // Add all of the input DATs into one huge internal DAT DatFile datdata = DatFile.Create(); foreach (ParentablePath datfile in datfilePaths) { Parser.ParseInto(datdata, datfile, int.MaxValue, keep: true); } // Set depot information datdata.Header.InputDepot = Header.InputDepot?.Clone() as DepotInformation; datdata.Header.OutputDepot = Header.OutputDepot?.Clone() as DepotInformation; // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) { datdata.Header.HeaderSkipper = Header.HeaderSkipper; } watch.Stop(); // If we have the depot flag, respect it bool success; if (Header.InputDepot?.IsActive ?? false) { success = Rebuilder.RebuildDepot(datdata, Inputs, OutputDir, date, delete, inverse, outputFormat); } else { success = Rebuilder.RebuildGeneric(datdata, Inputs, OutputDir, quickScan, date, delete, inverse, outputFormat, asFiles); } // If we have a success and we're updating the DAT, write it out if (success && updateDat) { datdata.Header.FileName = $"fixDAT_{Header.FileName}"; datdata.Header.Name = $"fixDAT_{Header.Name}"; datdata.Header.Description = $"fixDAT_{Header.Description}"; datdata.Items.ClearMarked(); Writer.Write(datdata, OutputDir); } } return(true); }
static void Main(string[] args) { Console.WriteLine("DAT Tool by CaptainSwag101\n" + "Version 0.0.1, built on 2019-10-22\n"); if (args.Length == 0) { Console.WriteLine("ERROR: No targets specified."); return; } foreach (string arg in args) { FileInfo info = new FileInfo(arg); if (!info.Exists) { Console.WriteLine($"ERROR: File \"{arg}\" does not exist, skipping."); continue; } if (info.Extension.ToLowerInvariant() == ".dat") { // Convert DAT to CSV DatFile dat = new DatFile(); dat.Load(info.FullName); StringBuilder output = new StringBuilder(); // Write header List <string> headerEntries = new List <string>(); foreach (var value in dat.ValueInfo) { headerEntries.Add($"{value.Name} ({value.Type})"); } output.AppendJoin(',', headerEntries); output.Append('\n'); // Write struct entries List <string> structEntries = new List <string>(); foreach (var entry in dat.StructEntries) { StringBuilder combinedEntry = new StringBuilder(); combinedEntry.AppendJoin(',', entry); structEntries.Add(combinedEntry.ToString()); } output.AppendJoin('\n', structEntries); using StreamWriter writer = new StreamWriter(info.FullName.TrimEnd(info.Extension.ToCharArray()) + ".csv", false); writer.Write(output.ToString()); } else if (info.Extension.ToLowerInvariant() == ".csv") { // Convert CSV to DAT DatFile dat = new DatFile(); using StreamReader reader = new StreamReader(info.FullName); // First line is header string[] header = reader.ReadLine().Split(','); List <(string Name, string Type)> valInfo = new List <(string Name, string Type)>(); foreach (string headerPiece in header) { string name = headerPiece.Split('(').First(); string type = headerPiece.Split('(').Last().TrimEnd(')'); valInfo.Add((name, type)); } dat.ValueInfo = valInfo; // Read struct entries List <List <string> > entries = new List <List <string> >(); while (!reader.EndOfStream) { List <string> entry = new List <string>(); entry.AddRange(reader.ReadLine().Split(',')); entries.Add(entry); } dat.StructEntries = entries; dat.Save(info.FullName.TrimEnd(info.Extension.ToCharArray()) + ".dat"); } else { Console.WriteLine($"ERROR: Invalid file extension \"{info.Extension}\"."); continue; } } }
/// <summary> /// Wrap sorting files using an input DAT /// </summary> /// <param name="datfiles">Names of the DATs to compare against</param> /// <param name="inputs">List of input files/folders to check</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="depot">True if the input direcories are treated as romba depots, false otherwise</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param> /// <param name="sevenzip">Integer representing the archive handling level for 7z</param> /// <param name="gz">Integer representing the archive handling level for GZip</param> /// <param name="rar">Integer representing the archive handling level for RAR</param> /// <param name="zip">Integer representing the archive handling level for Zip</param> /// <param name="updateDat">True if the updated DAT should be output, false otherwise</param> /// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param> /// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param> /// <param name="chdsAsFiles">True if CHDs should be treated like regular files, false otherwise</param> /// <param name="individual">True if DATs should be sorted individually, false if they should be done in bulk</param> private static void InitSort( List <string> datfiles, List <string> inputs, string outDir, bool depot, bool quickScan, bool date, bool delete, bool inverse, OutputFormat outputFormat, bool romba, int sevenzip, int gz, int rar, int zip, bool updateDat, string headerToCheckAgainst, SplitType splitType, bool chdsAsFiles, bool individual) { // Get the archive scanning level ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip); // Get a list of files from the input datfiles datfiles = Utilities.GetOnlyFilesFromInputs(datfiles); // If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir if (individual) { foreach (string datfile in datfiles) { DatFile datdata = new DatFile(); datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); // If we have the depot flag, respect it if (depot) { datdata.RebuildDepot(inputs, Path.Combine(outDir, datdata.FileName), date, delete, inverse, outputFormat, romba, updateDat, headerToCheckAgainst); } else { datdata.RebuildGeneric(inputs, Path.Combine(outDir, datdata.FileName), quickScan, date, delete, inverse, outputFormat, romba, asl, updateDat, headerToCheckAgainst, chdsAsFiles); } } } // Otherwise, process all DATs into the same output else { InternalStopwatch watch = new InternalStopwatch("Populating internal DAT"); // Add all of the input DATs into one huge internal DAT DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); } watch.Stop(); // If we have the depot flag, respect it if (depot) { datdata.RebuildDepot(inputs, outDir, date, delete, inverse, outputFormat, romba, updateDat, headerToCheckAgainst); } else { datdata.RebuildGeneric(inputs, outDir, quickScan, date, delete, inverse, outputFormat, romba, asl, updateDat, headerToCheckAgainst, chdsAsFiles); } } }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> public SabreXML(DatFile datFile) : base(datFile) { }
void createThumbnails() { #region array assignment thumbs[0] = thmb0; thumbs[1] = thmb1; thumbs[2] = thmb2; thumbs[3] = thmb3; thumbs[4] = thmb4; thumbs[5] = thmb5; thumbs[6] = thmb6; thumbs[7] = thmb7; thumbs[8] = thmb8; thumbs[9] = thmb9; thumbs[10] = thmb10; thumbs[11] = thmb11; thumbs[12] = thmb12; thumbs[13] = thmb13; thumbs[14] = thmb14; thumbs[15] = thmb15; thumbs[16] = thmb16; thumbs[17] = thmb17; thumbs[18] = thmb18; thumbs[19] = thmb19; thumbs[20] = thmb20; thumbs[21] = thmb21; thumbs[22] = thmb22; thumbs[23] = thmb23; thumbs[24] = thmb24; thumbs[25] = thmb25; thumbs[26] = thmb26; thumbs[27] = thmb27; thumbs[28] = thmb28; thumbs[29] = thmb29; thumbs[30] = thmb30; thumbs[31] = thmb31; thumbs[32] = thmb32; thumbs[33] = thmb33; thumbs[34] = thmb34; thumbs[35] = thmb35; thumbs[36] = thmb36; thumbs[37] = thmb37; thumbs[38] = thmb38; thumbs[39] = thmb39; thumbs[40] = thmb40; thumbs[41] = thmb41; thumbs[42] = thmb42; thumbs[43] = thmb43; thumbs[44] = thmb44; thumbs[45] = thmb45; thumbs[46] = thmb46; thumbs[47] = thmb47; thumbs[48] = thmb48; thumbs[49] = thmb49; thumbs[50] = thmb50; thumbs[51] = thmb51; thumbs[52] = thmb52; thumbs[53] = thmb53; thumbs[54] = thmb54; thumbs[55] = thmb55; thumbs[56] = thmb56; thumbs[57] = thmb57; thumbs[58] = thmb58; thumbs[59] = thmb59; thumbs[60] = thmb60; thumbs[61] = thmb61; thumbs[62] = thmb62; thumbs[63] = thmb63; thumbs[64] = thmb64; thumbs[65] = thmb65; thumbs[66] = thmb66; thumbs[67] = thmb67; thumbs[68] = thmb68; thumbs[69] = thmb69; thumbs[70] = thmb70; thumbs[71] = thmb71; thumbs[72] = thmb72; thumbs[73] = thmb73; thumbs[74] = thmb74; thumbs[75] = thmb75; thumbs[76] = thmb76; thumbs[77] = thmb77; thumbs[78] = thmb78; thumbs[79] = thmb79; thumbs[80] = thmb80; thumbs[81] = thmb81; thumbs[82] = thmb82; thumbs[83] = thmb83; thumbs[84] = thmb84; thumbs[85] = thmb85; thumbs[86] = thmb86; thumbs[87] = thmb87; thumbs[88] = thmb88; thumbs[89] = thmb89; thumbs[90] = thmb90; thumbs[91] = thmb91; thumbs[92] = thmb92; thumbs[93] = thmb93; thumbs[94] = thmb94; thumbs[95] = thmb95; thumbs[96] = thmb96; thumbs[97] = thmb97; thumbs[98] = thmb98; thumbs[99] = thmb99; thumbs[100] = thmb100; thumbs[101] = thmb101; thumbs[102] = thmb102; #endregion for (int i = 0; i < 103; i++) { thumbs[i].Tag = i; thumbs[i].BackgroundImageLayout = ImageLayout.Zoom; } // TODO: also look for customs in TIE-BoP if (_platform == MissionFile.Platform.TIE || _platform == MissionFile.Platform.XvT) { for (int i = 1; i <= 8; i++) { setThumbnail(_backdropDirectory + "PLANET" + i.ToString() + ".ACT", i - 1); } } else if (_platform == MissionFile.Platform.BoP) { for (int i = 1; i <= 8; i++) { setThumbnail(_backdropDirectory.Remove(_backdropDirectory.Length - 24, 15) + "PLANET" + i.ToString() + ".ACT", i - 1); } for (int i = 9; i <= 12; i++) { setThumbnail(_backdropDirectory + "PLANET" + i.ToString() + ".ACT", i - 1); } setThumbnail(_backdropDirectory + "SUN1B.ACT", 12); setThumbnail(_backdropDirectory + "SUN1C.ACT", 13); setThumbnail(_backdropDirectory + "SUN2B.ACT", 14); setThumbnail(_backdropDirectory + "SUN3A.ACT", 15); setThumbnail(_backdropDirectory + "SUN4A.ACT", 16); } else { thumbs[24].Enabled = false; _planets = new DatFile(); _planets.Groups.AutoSort = false; setThumbnail("planet.dat", 0, 24); _planets.Groups.Add(-1); setThumbnail("planet.dat", 25, 35); setThumbnail("wrapback.dat", 60, 2); setThumbnail("dsfire.dat", 62, 1); setThumbnail("nebula.dat", 63, 10); setThumbnail("galaxy.dat", 73, 10); setThumbnail("backdrop.dat", 83, 11); setThumbnail("wrapback.dat", 94, 4); setThumbnail("nebula.dat", 98, 5); for (int i = 0; i < _planets.NumberOfGroups; i++) { if (i == 24) { continue; } thumbs[i].BackgroundImage = _planets.Groups[i].Subs[0].Image; thumbs[i].BackColor = System.Drawing.Color.Black; } StreamReader sr; try //[JB Added try block { sr = new StreamReader(_installDirectory + "\\RESDATA.TXT"); } catch { MessageBox.Show("Could not open resource file:\n" + _installDirectory + "\\RESDATA.TXT", "Error"); return; } System.Collections.Generic.List <string> resdata = new System.Collections.Generic.List <string>(50); string line = ""; while ((line = sr.ReadLine()) != null) { if (line != "") { resdata.Add(line); } } for (int i = 0; i < resdata.Count - 38; i++) { DatFile temp = new DatFile(_installDirectory + "\\" + resdata[i]); int index = 0; for (int g = 0; g < temp.NumberOfGroups; g++) { index = _planets.Groups.GetIndex(temp.Groups[g].ID); if (index != -1) { _planets.Groups[index] = temp.Groups[g]; thumbs[index].BackgroundImage = temp.Groups[g].Subs[0].Image; } } } } }
/// <summary> /// Replace item values from the base set represented by the current DAT /// </summary> /// <param name="datFile">Current DatFile object to use for updating</param> /// <param name="intDat">DatFile to replace the values in</param> /// <param name="machineFields">List of MachineFields representing what should be updated</param> /// <param name="datItemFields">List of DatItemFields representing what should be updated</param> /// <param name="onlySame">True if descriptions should only be replaced if the game name is the same, false otherwise</param> public static void BaseReplace( DatFile datFile, DatFile intDat, List <MachineField> machineFields, List <DatItemField> datItemFields, bool onlySame) { InternalStopwatch watch = new InternalStopwatch($"Replacing items in '{intDat.Header.FileName}' from the base DAT"); // If we are matching based on DatItem fields of any sort if (datItemFields.Any()) { // For comparison's sake, we want to use CRC as the base bucketing datFile.Items.BucketBy(ItemKey.CRC, DedupeType.Full); intDat.Items.BucketBy(ItemKey.CRC, DedupeType.None); // Then we do a hashwise comparison against the base DAT Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key => { ConcurrentList <DatItem> datItems = intDat.Items[key]; ConcurrentList <DatItem> newDatItems = new ConcurrentList <DatItem>(); foreach (DatItem datItem in datItems) { ConcurrentList <DatItem> dupes = datFile.Items.GetDuplicates(datItem, sorted: true); DatItem newDatItem = datItem.Clone() as DatItem; // Replace fields from the first duplicate, if we have one if (dupes.Count > 0) { Replacer.ReplaceFields(newDatItem, dupes.First(), datItemFields); } newDatItems.Add(newDatItem); } // Now add the new list to the key intDat.Items.Remove(key); intDat.Items.AddRange(key, newDatItems); }); } // If we are matching based on Machine fields of any sort if (machineFields.Any()) { // For comparison's sake, we want to use Machine Name as the base bucketing datFile.Items.BucketBy(ItemKey.Machine, DedupeType.Full); intDat.Items.BucketBy(ItemKey.Machine, DedupeType.None); // Then we do a namewise comparison against the base DAT Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key => { ConcurrentList <DatItem> datItems = intDat.Items[key]; ConcurrentList <DatItem> newDatItems = new ConcurrentList <DatItem>(); foreach (DatItem datItem in datItems) { DatItem newDatItem = datItem.Clone() as DatItem; if (datFile.Items.ContainsKey(key) && datFile.Items[key].Count() > 0) { Replacer.ReplaceFields(newDatItem.Machine, datFile.Items[key][0].Machine, machineFields, onlySame); } newDatItems.Add(newDatItem); } // Now add the new list to the key intDat.Items.Remove(key); intDat.Items.AddRange(key, newDatItems); }); } watch.Stop(); }
/// <summary> /// Populate from multiple paths while returning the invividual headers /// </summary> /// <param name="datFile">Current DatFile object to use for updating</param> /// <param name="inputs">Paths to DATs to parse</param> /// <returns>List of DatHeader objects representing headers</returns> public static List <DatHeader> PopulateUserData(DatFile datFile, List <string> inputs) { List <ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList(); return(PopulateUserData(datFile, paths)); }
/// <summary> /// Wrap verifying files using an input DAT /// </summary> /// <param name="datfiles">Names of the DATs to compare against</param> /// <param name="inputs">Input directories to compare against</param> /// <param name="depot">True if the input direcories are treated as romba depots, false otherwise</param> /// <param name="hashOnly">True if only hashes should be checked, false for full file information</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param> /// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param> /// <param name="chdsAsFiles">True if CHDs should be treated like regular files, false otherwise</param> /// <param name="individual">True if DATs should be verified individually, false if they should be done in bulk</param> /// <param name="filter">Filter object to be passed to the DatItem level</param> private static void InitVerify( List <string> datfiles, List <string> inputs, bool depot, bool hashOnly, bool quickScan, string headerToCheckAgainst, SplitType splitType, bool chdsAsFiles, bool individual, Filter filter) { // Get the archive scanning level ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); // Get a list of files from the input datfiles datfiles = Utilities.GetOnlyFilesFromInputs(datfiles); // If we are in individual mode, process each DAT on their own if (individual) { foreach (string datfile in datfiles) { DatFile datdata = new DatFile(); datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); // If we have the depot flag, respect it if (depot) { datdata.VerifyDepot(inputs, headerToCheckAgainst); } else { datdata.VerifyGeneric(inputs, hashOnly, quickScan, headerToCheckAgainst, chdsAsFiles, filter); } } } // Otherwise, process all DATs into the same output else { InternalStopwatch watch = new InternalStopwatch("Populating internal DAT"); // Add all of the input DATs into one huge internal DAT DatFile datdata = new DatFile(); foreach (string datfile in datfiles) { datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true); } watch.Stop(); // If we have the depot flag, respect it if (depot) { datdata.VerifyDepot(inputs, headerToCheckAgainst); } else { datdata.VerifyGeneric(inputs, hashOnly, quickScan, headerToCheckAgainst, chdsAsFiles, filter); } } }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> /// <param name="quotes">Enable quotes on read and write, false otherwise</param> public ClrMamePro(DatFile datFile, bool quotes) : base(datFile) { Quotes = quotes; }
/// <summary> /// Apply cleaning methods to the DatFile /// </summary> /// <param name="datFile">Current DatFile object to run operations on</param> /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param> /// <returns>True if cleaning was successful, false on error</returns> public bool ApplyCleaning(DatFile datFile, bool throwOnError = false) { InternalStopwatch watch = new InternalStopwatch("Applying cleaning steps to DAT"); try { // Perform item-level cleaning CleanDatItems(datFile); // Bucket and dedupe according to the flag if (DedupeRoms == DedupeType.Full) { datFile.Items.BucketBy(ItemKey.CRC, DedupeRoms); } else if (DedupeRoms == DedupeType.Game) { datFile.Items.BucketBy(ItemKey.Machine, DedupeRoms); } // Process description to machine name if (DescriptionAsName == true) { MachineDescriptionToName(datFile); } // If we are removing scene dates, do that now if (SceneDateStrip == true) { StripSceneDatesFromItems(datFile); } // Run the one rom per game logic, if required if (OneGamePerRegion == true) { SetOneGamePerRegion(datFile); } // Run the one rom per game logic, if required if (OneRomPerGame == true) { SetOneRomPerGame(datFile); } // Remove all marked items datFile.Items.ClearMarked(); // We remove any blanks, if we aren't supposed to have any if (KeepEmptyGames == false) { datFile.Items.ClearEmpty(); } } catch (Exception ex) when(!throwOnError) { logger.Error(ex); return(false); } finally { watch.Stop(); } return(true); }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> public OfflineList(DatFile datFile) : base(datFile) { }
/// <summary> /// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof /// </summary> /// <param name="datFile">Current DatFile object to run operations on</param> /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param> internal void MachineDescriptionToName(DatFile datFile, bool throwOnError = false) { try { // First we want to get a mapping for all games to description ConcurrentDictionary <string, string> mapping = new ConcurrentDictionary <string, string>(); Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { ConcurrentList <DatItem> items = datFile.Items[key]; foreach (DatItem item in items) { // If the key mapping doesn't exist, add it mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''").Replace(":", " -")); } }); // Now we loop through every item and update accordingly Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key => { ConcurrentList <DatItem> items = datFile.Items[key]; ConcurrentList <DatItem> newItems = new ConcurrentList <DatItem>(); foreach (DatItem item in items) { // Update machine name if (!string.IsNullOrWhiteSpace(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name)) { item.Machine.Name = mapping[item.Machine.Name]; } // Update cloneof if (!string.IsNullOrWhiteSpace(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf)) { item.Machine.CloneOf = mapping[item.Machine.CloneOf]; } // Update romof if (!string.IsNullOrWhiteSpace(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf)) { item.Machine.RomOf = mapping[item.Machine.RomOf]; } // Update sampleof if (!string.IsNullOrWhiteSpace(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf)) { item.Machine.SampleOf = mapping[item.Machine.SampleOf]; } // Add the new item to the output list newItems.Add(item); } // Replace the old list of roms with the new one datFile.Items.Remove(key); datFile.Items.AddRange(key, newItems); }); } catch (Exception ex) when(!throwOnError) { logger.Warning(ex.ToString()); } }
/// <summary> /// Attempt to add a file to the output if it matches /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="file">Name of the file to process</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>True if the file was used to rebuild, false otherwise</returns> private static bool RebuildGenericHelper( DatFile datFile, string file, string outDir, bool quickScan, bool date, bool inverse, OutputFormat outputFormat, TreatAsFile asFiles) { // If we somehow have a null filename, return if (file == null) { return(false); } // Set the deletion variables bool usedExternally = false, usedInternally = false; // Create an empty list of BaseFile for archive entries List <BaseFile> entries = null; // Get the TGZ and TXZ status for later GZipArchive tgz = new GZipArchive(file); XZArchive txz = new XZArchive(file); bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent(); // Get the base archive first BaseArchive archive = BaseArchive.Create(file); // Now get all extracted items from the archive if (archive != null) { archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard; entries = archive.GetChildren(); } // If the entries list is null, we encountered an error or have a file and should scan externally if (entries == null && File.Exists(file)) { BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles); // Create the correct DatItem DatItem internalDatItem; if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat)) { internalDatItem = new Media(internalFileInfo); } else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) { internalDatItem = new Disk(internalFileInfo); } else { internalDatItem = new Rom(internalFileInfo); } usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat); } // Otherwise, loop through the entries and try to match else { foreach (BaseFile entry in entries) { DatItem internalDatItem = DatItem.Create(entry); usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */); } } return(usedExternally || usedInternally); }
/// <summary> /// Filter a DAT using 1G1R logic given an ordered set of regions /// </summary> /// <param name="datFile">Current DatFile object to run operations on</param> /// <remarks> /// In the most technical sense, the way that the region list is being used does not /// confine its values to be just regions. Since it's essentially acting like a /// specialized version of the machine name filter, anything that is usually encapsulated /// in parenthesis would be matched on, including disc numbers, languages, editions, /// and anything else commonly used. Please note that, unlike other existing 1G1R /// solutions, this does not have the ability to contain custom mappings of parent /// to clone sets based on name, nor does it have the ability to match on the /// Release DatItem type. /// </remarks> internal void SetOneGamePerRegion(DatFile datFile) { // If we have null region list, make it empty if (RegionList == null) { RegionList = new List <string>(); } // For sake of ease, the first thing we want to do is bucket by game datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); // Then we want to get a mapping of all machines to parents Dictionary <string, List <string> > parents = new Dictionary <string, List <string> >(); foreach (string key in datFile.Items.Keys) { DatItem item = datFile.Items[key][0]; // Match on CloneOf first if (!string.IsNullOrEmpty(item.Machine.CloneOf)) { if (!parents.ContainsKey(item.Machine.CloneOf.ToLowerInvariant())) { parents.Add(item.Machine.CloneOf.ToLowerInvariant(), new List <string>()); } parents[item.Machine.CloneOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); } // Then by RomOf else if (!string.IsNullOrEmpty(item.Machine.RomOf)) { if (!parents.ContainsKey(item.Machine.RomOf.ToLowerInvariant())) { parents.Add(item.Machine.RomOf.ToLowerInvariant(), new List <string>()); } parents[item.Machine.RomOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); } // Otherwise, treat it as a parent else { if (!parents.ContainsKey(item.Machine.Name.ToLowerInvariant())) { parents.Add(item.Machine.Name.ToLowerInvariant(), new List <string>()); } parents[item.Machine.Name.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant()); } } // Once we have the full list of mappings, filter out games to keep foreach (string key in parents.Keys) { // Find the first machine that matches the regions in order, if possible string machine = default; foreach (string region in RegionList) { machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase)); if (machine != default) { break; } } // If we didn't get a match, use the parent if (machine == default) { machine = key; } // Remove the key from the list parents[key].Remove(machine); // Remove the rest of the items from this key parents[key].ForEach(k => datFile.Items.Remove(k)); } // Finally, strip out the parent tags Splitter.RemoveTagsFromChild(datFile); }
/// <summary> /// Process the DAT and find all matches in input files and folders assuming they're a depot /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="inputs">List of input files/folders to check</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <returns>True if rebuilding was a success, false otherwise</returns> public static bool RebuildDepot( DatFile datFile, List <string> inputs, string outDir, bool date = false, bool delete = false, bool inverse = false, OutputFormat outputFormat = OutputFormat.Folder) { #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit if (datFile.Items.TotalCount == 0 && !inverse) { logger.User("No entries were found to rebuild, exiting..."); return(false); } // Check that the output directory exists outDir = outDir.Ensure(create: true); // Now we want to get forcepack flag if it's not overridden if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None) { outputFormat = GetOutputFormat(datFile.Header.ForcePacking); } #endregion bool success = true; #region Rebuild from depots in order string format = FromOutputFormat(outputFormat) ?? string.Empty; InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}"); // Now loop through and get only directories from the input paths List <string> directories = new List <string>(); Parallel.ForEach(inputs, Globals.ParallelOptions, input => { // Add to the list if the input is a directory if (Directory.Exists(input)) { logger.Verbose($"Adding depot: {input}"); lock (directories) { directories.Add(input); } } }); // If we don't have any directories, we want to exit if (directories.Count == 0) { return(success); } // Now that we have a list of depots, we want to bucket the input DAT by SHA-1 datFile.Items.BucketBy(ItemKey.SHA1, DedupeType.None); // Then we want to loop through each of the hashes and see if we can rebuild var keys = datFile.Items.SortedKeys.ToList(); foreach (string hash in keys) { // Pre-empt any issues that could arise from string length if (hash.Length != Constants.SHA1Length) { continue; } logger.User($"Checking hash '{hash}'"); // Get the extension path for the hash string subpath = Utilities.GetDepotPath(hash, datFile.Header.InputDepot.Depth); // Find the first depot that includes the hash string foundpath = null; foreach (string directory in directories) { if (File.Exists(Path.Combine(directory, subpath))) { foundpath = Path.Combine(directory, subpath); break; } } // If we didn't find a path, then we continue if (foundpath == null) { continue; } // If we have a path, we want to try to get the rom information GZipArchive archive = new GZipArchive(foundpath); BaseFile fileinfo = archive.GetTorrentGZFileInfo(); // If the file information is null, then we continue if (fileinfo == null) { continue; } // Ensure we are sorted correctly (some other calls can change this) datFile.Items.BucketBy(ItemKey.SHA1, DedupeType.None); // If there are no items in the hash, we continue if (datFile.Items[hash] == null || datFile.Items[hash].Count == 0) { continue; } // Otherwise, we rebuild that file to all locations that we need to bool usedInternally; if (datFile.Items[hash][0].ItemType == ItemType.Disk) { usedInternally = RebuildIndividualFile(datFile, new Disk(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */); } else if (datFile.Items[hash][0].ItemType == ItemType.Media) { usedInternally = RebuildIndividualFile(datFile, new Media(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */); } else { usedInternally = RebuildIndividualFile(datFile, new Rom(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */); } // If we are supposed to delete the depot file, do so if (delete && usedInternally) { File.Delete(foundpath); } } watch.Stop(); #endregion return(success); }
private bool LoadRomFromDat(DatFileLoader dfl, DatDir parentDir) { dfl.Gn(); if (dfl.Next != "(") { _errorReport?.Invoke(dfl.Filename, "( not found after rom, on line " + dfl.LineNumber); return(false); } dfl.Gn(); if (dfl.Next.ToLower() != "name") { _errorReport?.Invoke(dfl.Filename, "Name not found as first object in ( ), on line " + dfl.LineNumber); return(false); } DatFile dRom = new DatFile(DatFileType.UnSet) { Name = dfl.Gn() }; dfl.Gn(); while (dfl.Next != ")") { switch (dfl.Next.ToLower()) { case "size": dRom.Size = VarFix.ULong(dfl.Gn()); break; case "hash": dfl.Gn(); break; case "crc": dRom.CRC = VarFix.CleanMD5SHA1(dfl.Gn(), 8); break; case "sha1": dRom.SHA1 = VarFix.CleanMD5SHA1(dfl.Gn(), 40); break; case "md5": dRom.MD5 = VarFix.CleanMD5SHA1(dfl.Gn(), 32); break; case "merge": dRom.Merge = VarFix.String(dfl.Gn()); break; case "flags": string flags = VarFix.ToLower(dfl.Gn()); if (string.IsNullOrWhiteSpace(dRom.Status)) { dRom.Status = flags; } break; case "date": dfl.Gn(); break; case "bios": dfl.Gn(); break; case "region": dfl.Gn(); break; case "offs": dfl.Gn(); break; case "nodump": dRom.Status = "nodump"; break; case "baddump": dRom.Status = "baddump"; break; default: _errorReport?.Invoke(dfl.Filename, "Error: key word '" + dfl.Next + "' not known in rom, on line " + dfl.LineNumber); break; } dfl.Gn(); } parentDir.ChildAdd(dRom); return(true); }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> public ClrMamePro(DatFile datFile) : base(datFile, cloneHeader: false) { }
private bool LoadDiskFromDat(DatFileLoader dfl, DatDir parentDir) { dfl.Gn(); if (dfl.Next != "(") { _errorReport?.Invoke(dfl.Filename, "( not found after rom, on line " + dfl.LineNumber); return(false); } dfl.Gn(); if (dfl.Next.ToLower() != "name") { _errorReport?.Invoke(dfl.Filename, "Name not found as first object in ( ), on line " + dfl.LineNumber); return(false); } DatFile dRom = new DatFile(DatFileType.UnSet) { Name = VarFix.String(dfl.Gn()) + ".chd", isDisk = true }; dfl.Gn(); while (dfl.Next != ")") { switch (dfl.Next.ToLower()) { case "sha1": dRom.SHA1 = VarFix.CleanMD5SHA1(dfl.Gn(), 40); break; case "md5": dRom.MD5 = VarFix.CleanMD5SHA1(dfl.Gn(), 32); break; case "region": dRom.Region = VarFix.String(dfl.Gn()); break; case "merge": dRom.Merge = VarFix.String(dfl.Gn()); break; case "index": dfl.Gn(); break; case "flags": dRom.Status = VarFix.ToLower(dfl.Gn()); break; case "nodump": dRom.Status = "nodump"; break; default: _errorReport?.Invoke(dfl.Filename, "Error: key word '" + dfl.Next + "' not known in rom, on line " + dfl.LineNumber); break; } dfl.Gn(); } parentDir.ChildAdd(dRom); return(true); }
public Race(string filename, string playerVehicleFile) { Race.Current = this; Logger.Log("Starting race " + Path.GetFileName(filename)); ConfigFile = new RaceFile(filename); foreach (string matFileName in ConfigFile.MaterialFiles) { MatFile matFile = new MatFile(matFileName); ResourceCache.Add(matFile); } foreach (string pixFileName in ConfigFile.PixFiles) { PixFile pixFile = new PixFile(pixFileName); ResourceCache.Add(pixFile); } if (GameVars.Emulation == EmulationMode.Demo) { ResourceCache.Add(new CMaterial("drkcurb.mat", 226)); //demo doesn't have this file, I guess the color is hard-coded } else { ResourceCache.Add(new MatFile("drkcurb.mat")); } ResourceCache.ResolveMaterials(); if (filename.Contains("TESTMAP")) //nasty hack... { GameVars.Scale.Y *= 0.5f; } DatFile modelFile = new DatFile(ConfigFile.ModelFile); ActFile actFile = new ActFile(ConfigFile.ActorFile); _actors = actFile.Hierarchy; _actors.AttachModels(modelFile.Models); _actors.ResolveTransforms(false, ConfigFile.Grooves); if (filename.Contains("TESTMAP")) //nasty hack... { GameVars.Scale.Y *= 2f; } // link the actors and grooves foreach (BaseGroove g in ConfigFile.Grooves) { g.SetActor(_actors.GetByName(g.ActorName)); } // link the funks and materials foreach (BaseFunk f in ConfigFile.Funks) { f.Resolve(); } if (ConfigFile.SkyboxTexture != "none") { PixFile horizonPix = new PixFile(ConfigFile.SkyboxTexture); _skybox = SkyboxGenerator.Generate(horizonPix.PixMaps[0].Texture, ConfigFile.SkyboxRepetitionsX - 3f, ConfigFile.DepthCueMode); _skybox.HeightOffset = -220 + ConfigFile.SkyboxPositionY * 1.5f; } Physics.TrackProcessor.GenerateTrackActor(ConfigFile, _actors, out _nonCars); Logger.Log("NonCars: " + _nonCars.Count); GridPlacer.Reset(); List <int> opponentIds = new List <int>(); List <int> pickedNbrs = new List <int>(); for (int i = 0; i < 5; i++) { int index = 0; while (true) { index = Engine.Random.Next(1, OpponentsFile.Instance.Opponents.Count); if (!pickedNbrs.Contains(index)) { pickedNbrs.Add(index); break; } } try { Opponents.Add(new Opponent(OpponentsFile.Instance.Opponents[index].FileName, ConfigFile.GridPosition, ConfigFile.GridDirection)); NbrOpponents++; } catch (Exception ex) { Logger.Log("Error while loading opponent " + OpponentsFile.Instance.Opponents[index].FileName + ", " + ex.Message); } } foreach (CopStartPoint point in ConfigFile.CopStartPoints) { Opponents.Add(new Opponent(point.IsSpecialForces ? "bigapc.txt" : "apc.txt", point.Position, 0, new CopDriver())); } foreach (Opponent o in Opponents) { Drivers.Add(o.Driver); } OpponentController.Nodes = ConfigFile.OpponentPathNodes; PlayerVehicle = new Vehicle(GameVars.BasePath + @"cars\" + playerVehicleFile, new PlayerDriver()); PlayerVehicle.PlaceOnGrid(ConfigFile.GridPosition, ConfigFile.GridDirection); Drivers.Add(PlayerVehicle.Driver); Peds = new PedestrianController(ConfigFile.Peds); _map = new RaceMap(this); RaceTime = new RaceTimeController(); PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.TrackId, PhysXConsts.VehicleId, ContactPairFlag.Forces | ContactPairFlag.OnStartTouch | ContactPairFlag.OnTouch); PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.VehicleId, PhysXConsts.NonCarId, ContactPairFlag.Forces | ContactPairFlag.OnStartTouch | ContactPairFlag.OnTouch); PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.TrackId, PhysXConsts.NonCarId, ContactPairFlag.OnTouch); PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.VehicleId, PhysXConsts.VehicleId, ContactPairFlag.Forces | ContactPairFlag.OnTouch | ContactPairFlag.OnStartTouch | ContactPairFlag.OnEndTouch); }
/// <summary> /// Constructor designed for casting a base DatFile /// </summary> /// <param name="datFile">Parent DatFile to copy from</param> /// <param name="deprecated">True if the output uses "game", false if the output uses "machine"</param> public Logiqx(DatFile datFile, bool deprecated) : base(datFile) { _deprecated = deprecated; }
private bool LoadDisks(DatFileLoader dfl, DatDir parentDir) { if (dfl.Next.ToLower() != "[disks]") { _errorReport?.Invoke(dfl.Filename, "Looking for [DISKS] but found " + dfl.Next + " , " + dfl.LineNumber); return(false); } while (!dfl.EndOfStream()) { string line = dfl.Gn(); if (line.Substring(0, 1) == "[") { return(true); } string[] parts = line.Split('¬'); // 1 parent name = clone of // 2 parent description = description (from parent) // 3 game name = name (game) // 4 game description = description // 5 rom name = name (rom) // 6 rom crc = crc // 7 rom size = size // 8 romof name = romof // 9 merge name = merge string ParentName = parts[1]; string ParentDescription = parts[2]; string GameName = parts[3]; string GameDescription = parts[4]; string romName = parts[5]; string romCRC = parts[6]; string romSize = parts[7]; string romOf = parts[8]; string merge = parts[9]; int index; DatDir dDir; DatDir searchDir = new DatDir(DatFileType.Dir) { Name = GameName }; if (parentDir.ChildNameSearch(searchDir, out index) != 0) { dDir = new DatDir(DatFileType.UnSet) { Name = GameName, DGame = new DatGame() }; DatGame dGame = dDir.DGame; dGame.Description = GameDescription; if (ParentName != GameName) { dGame.CloneOf = ParentName; } parentDir.ChildAdd(dDir); } else { dDir = (DatDir)parentDir.Child(index); // need to check everything matches } DatFile dRom = new DatFile(DatFileType.UnSet) { isDisk = true, Name = romName, SHA1 = VarFix.CleanMD5SHA1(romCRC, 40), Merge = merge }; // dRom.Size = VarFix.ULong(romSize); // check romof=ParentName dDir.ChildAdd(dRom); } return(true); }
public void Start() { dispatcher = new Dispatcher(); scheduler = new Scheduler(dispatcher); listeners = new List <Listener>(); listeners.Add(new Listener(7171, socket => new LoginConnection(this, socket))); listeners.Add(new Listener(7172, socket => new GameConnection(this, socket))); Clock = new Clock(12, 0); Logger = new Logger(); Channels = new ChannelCollection(); RuleViolations = new RuleViolationCollection(); PacketsFactory = new PacketsFactory(); using (Logger.Measure("Loading items", true)) { var otb = OtbFile.Load("data/items/items.otb"); var dat = DatFile.Load("data/items/tibia.dat"); var items = ItemsFile.Load("data/items/items.xml"); ItemFactory = new ItemFactory(otb, dat, items); } using (Logger.Measure("Loading monsters", true)) { var monsters = MonsterFile.Load("data/monsters"); MonsterFactory = new MonsterFactory(monsters); } using (Logger.Measure("Loading npcs", true)) { var npcs = NpcFile.Load("data/npcs"); NpcFactory = new NpcFactory(npcs); } using (Logger.Measure("Loading map", true)) { var otbm = OtbmFile.Load("data/map/pholium3.otbm"); Map = new Map(ItemFactory, otbm); } Pathfinding = new Pathfinding(Map); Events = new EventsCollection(); Scripts = new ScriptsCollection(); using (Logger.Measure("Loading scripts", true)) { Scripts.Start(this); } dispatcher.Start(); scheduler.Start(); foreach (var listener in listeners) { listener.Start(); } Logger.WriteLine("Server online"); }
private int InternalHandle(DatFile file) { return (int)typeof(DatFile).GetProperty("Handle", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(file, null); }
/// <summary> /// Output non-duplicate item diff /// </summary> /// <param name="datFile">Current DatFile object to use for updating</param> /// <param name="inputs">List of inputs to write out from</param> public static DatFile DiffNoDuplicates(DatFile datFile, List <string> inputs) { List <ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList(); return(DiffNoDuplicates(datFile, paths)); }