public MapFile Read(DatFile dataFileReference = null)
{
List <IGenericFieldBlock> fields = new List <IGenericFieldBlock>();
using (FileStream fs = new FileStream(this.infile, FileMode.Open, FileAccess.Read))
{
using (BinaryReader r = new BinaryReader(fs))
{
// construct an object using the file spec
char[] fileID = r.ReadChars(4);
if (string.Compare(new string(fileID), new string(MAGIC)) != 0)
{
throw new ArgumentException("Not a valid Army Men 1 map file");
}
// checknum
UInt32 fileContentSizeBytes = r.ReadUInt32(); // not including magic id and file size
_ = r.ReadChars(4); // "MAP " identifier
// I dont know that this is correct way to read the version
int major = 6;
int minor = 0;
while (r.BaseStream.Position != r.BaseStream.Length)
{
char[] fieldID = r.ReadChars(4);
if (r.BaseStream.Position == r.BaseStream.Length) // some files appear to have an extra nil at the end
{
break;
}
// there seems to be a bug in the OATT field length int, so we need to parse it bit by bit
string id = new string(fieldID);
Int32 size;
switch (id)
{
case "TLAY":
TNAMBlock tNAMBlock = (TNAMBlock)fields.FindLast(x => x.DisplayFieldName == "Textures");
fields.Add(new TLAYBlock(r, tNAMBlock));
break;
case "TNAM":
fields.Add(new TNAMBlock(r, Directory.GetParent(this.infile).FullName));
break;
case "OLAY":
fields.Add(new OLAYBlock(r));
break;
case "OATT":
OLAYBlock lastOLAY = (OLAYBlock)fields.FindLast(x => x.DisplayFieldName == OLAYBlock.FIELD_NAME);
fields.Add(new OATTBlock(r, lastOLAY));
break;
case "SCEN":
fields.Add(new SCENBlock(r));
break;
case "VERS":
size = r.ReadInt32(); // length of version field. Always 8.
// this field always occurs either first or after CNUM. CNUM doesnt always appear.
major = r.ReadInt32();
minor = r.ReadInt32();
Span <byte> content = stackalloc byte[size];
BinaryPrimitives.WriteInt32LittleEndian(content.Slice(0, 4), major);
BinaryPrimitives.WriteInt32LittleEndian(content.Slice(4, 4), minor);
fields.Add(new GenericFieldBlock(fieldID, size, content.ToArray()));
break;
default:
size = r.ReadInt32();
{
byte[] c = r.ReadBytes(size);
fields.Add(new GenericFieldBlock(fieldID, size, c));
}
break;
}
}
}
}
fields.Add(
new CombinedTextureLayersBlock(
fields.OfType <TNAMBlock>().First(),
fields
.OfType <TLAYBlock>()
.ToList(),
fields.OfType <OLAYBlock>().ToList(),
dataFileReference
)
);
return(new MapFile(fields));
}
/// <summary>
/// Wrap adding files to the depots
/// </summary>
/// <param name="inputs">List of input folders to use</param>
/// <param name="onlyNeeded">True if only files in the database and don't exist are added, false otherwise</param>
/// <param name="resume">Resume a previously interrupted operation from the specified path</param>
/// <param name="includeZips">flag value == 0 means: add Zip files themselves into the depot in addition to their contents, flag value == 2 means add Zip files themselves but don't add content</param>
/// <param name="workers">How many workers to launch for the job, default from config</param>
/// <param name="includeGZips">flag value == 0 means: add GZip files themselves into the depot in addition to their contents, flag value == 2 means add GZip files themselves but don't add content</param>
/// <param name="include7Zips">flag value == 0 means: add 7Zip files themselves into the depot in addition to their contents, flag value == 2 means add 7Zip files themselves but don't add content</param>
/// <param name="skipInitialScan">True to skip the initial scan of the files to determine amount of work, false otherwise</param>
/// <param name="useGolangZip">True to use go zip implementation instead of zlib, false otherwise</param>
/// <param name="noDb">True to archive into depot but do not touch DB index and ignore only-needed flag, false otherwise</param>
/// TODO: Add ability to update .romba files with proper size AND use the correct depot if it fills up
/// TODO: Add ability correctly to mark which depot the files are being rebuilt to in the DB
private static void InitArchive(
List <string> inputs,
bool onlyNeeded,
string resume,
int includeZips,
int workers,
int includeGZips,
int include7Zips,
bool skipInitialScan,
bool useGolangZip, // Obsolete
bool noDb)
{
// First we want to get just all directories from the inputs
List <string> onlyDirs = new List <string>();
foreach (string input in inputs)
{
if (Directory.Exists(input))
{
onlyDirs.Add(Path.GetFullPath(input));
}
}
// Then process all of the input directories into an internal DAT
DatFile df = new DatFile();
foreach (string dir in onlyDirs)
{
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
df.PopulateFromDir(dir, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true);
df.PopulateFromDir(dir, Hash.DeepHashes, false, true, SkipFileType.None, false, false, _tmpdir, false, null, true);
}
// Create an empty Dat for files that need to be rebuilt
DatFile need = new DatFile();
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now that we have the Dats, add the files to the database
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
foreach (string key in df.Keys)
{
List <DatItem> datItems = df[key];
foreach (Rom rom in datItems)
{
// If we care about if the file exists, check the databse first
if (onlyNeeded && !noDb)
{
string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"
+ " WHERE crcsha1.crc=\"" + rom.CRC + "\""
+ " OR md5sha1.md5=\"" + rom.MD5 + "\""
+ " OR md5sha1.sha1=\"" + rom.SHA1 + "\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
// Add to the queries
if (!String.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += " (\"" + rom.CRC + "\"),";
}
if (!String.IsNullOrWhiteSpace(rom.MD5))
{
md5query += " (\"" + rom.MD5 + "\"),";
}
if (!String.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += " (\"" + rom.SHA1 + "\", \"" + _depots.Keys.ToList()[0] + "\"),";
if (!String.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),";
}
if (!String.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),";
}
}
// Add to the Dat
need.Add(key, rom);
}
}
// Otherwise, just add the file to the list
else
{
// Add to the queries
if (!noDb)
{
if (!String.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += " (\"" + rom.CRC + "\"),";
}
if (!String.IsNullOrWhiteSpace(rom.MD5))
{
md5query += " (\"" + rom.MD5 + "\"),";
}
if (!String.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += " (\"" + rom.SHA1 + "\", \"" + _depots.Keys.ToList()[0] + "\"),";
if (!String.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),";
}
if (!String.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),";
}
}
}
// Add to the Dat
need.Add(key, rom);
}
}
}
// Now run the queries, if they're populated
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
// Create the sorting object to use and rebuild the needed files
ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(include7Zips, includeGZips, 2, includeZips);
need.RebuildGeneric(onlyDirs, _depots.Keys.ToList()[0], false /*quickScan*/, false /*date*/,
false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzip, true /*romba*/, asl, false /*updateDat*/,
null /*headerToCheckAgainst*/, true /* chdsAsFiles */);
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
public ArchiveDotOrg(DatFile datFile)
: base(datFile)
{
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
public AttractMode(DatFile datFile)
: base(datFile)
{
}
/// <summary>
/// Process the DAT and find all matches in input files and folders
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public static bool RebuildGeneric(
DatFile datFile,
List <string> inputs,
string outDir,
bool quickScan = false,
bool date = false,
bool delete = false,
bool inverse = false,
OutputFormat outputFormat = OutputFormat.Folder,
TreatAsFile asFiles = 0x00)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (datFile.Items.TotalCount == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return(false);
}
// Check that the output directory exists
if (!Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
outDir = Path.GetFullPath(outDir);
}
// Now we want to get forcepack flag if it's not overridden
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None)
{
outputFormat = GetOutputFormat(datFile.Header.ForcePacking);
}
#endregion
bool success = true;
#region Rebuild from sources in order
string format = FromOutputFormat(outputFormat) ?? string.Empty;
InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}");
// Now loop through all of the files in all of the inputs
foreach (string input in inputs)
{
// If the input is a file
if (File.Exists(input))
{
logger.User($"Checking file: {input}");
bool rebuilt = RebuildGenericHelper(datFile, input, outDir, quickScan, date, inverse, outputFormat, asFiles);
// If we are supposed to delete the file, do so
if (delete && rebuilt)
{
File.Delete(input);
}
}
// If the input is a directory
else if (Directory.Exists(input))
{
logger.Verbose($"Checking directory: {input}");
foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
{
logger.User($"Checking file: {file}");
bool rebuilt = RebuildGenericHelper(datFile, file, outDir, quickScan, date, inverse, outputFormat, asFiles);
// If we are supposed to delete the file, do so
if (delete && rebuilt)
{
File.Delete(input);
}
}
}
}
watch.Stop();
#endregion
return(success);
}
/// <summary>
/// Find duplicates and rebuild individual files to output
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
/// <returns>True if the file was able to be rebuilt, false otherwise</returns>
private static bool RebuildIndividualFile(
DatFile datFile,
DatItem datItem,
string file,
string outDir,
bool date,
bool inverse,
OutputFormat outputFormat,
bool?isZip = null)
{
// Set the initial output value
bool rebuilt = false;
// If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) &&
!(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) &&
!(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
outputFormat = OutputFormat.Folder;
}
// If we have a Disk or Media, change it into a Rom for later use
if (datItem.ItemType == ItemType.Disk)
{
datItem = (datItem as Disk).ConvertToRom();
}
else if (datItem.ItemType == ItemType.Media)
{
datItem = (datItem as Media).ConvertToRom();
}
// Prepopluate a key string
string crc = (datItem as Rom).CRC ?? string.Empty;
// Try to get the stream for the file
if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
{
return(false);
}
// If either we have duplicates or we're filtering
if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes))
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
// If we have a very specific TXZ->TXZ case, just copy it accordingly
if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Special case for partial packing mode
bool shouldCheck = false;
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
{
shouldCheck = true;
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false);
}
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// If we should check for the items in the machine
if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
{
outputFormat = OutputFormat.Folder;
}
else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
{
outputFormat = OutputFormat.ParentFolder;
}
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
}
// Close the input stream
fileStream?.Dispose();
}
// Now we want to take care of headers, if applicable
if (datFile.Header.HeaderSkipper != null)
{
// Check to see if we have a matching header first
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
MemoryStream transformStream = new MemoryStream();
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
{
logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// Create a headered item to use as well
datItem.CopyMachineInformation(item);
datItem.SetName($"{datItem.GetName()}_{crc}");
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
bool eitherSuccess = false;
eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
}
}
}
// Dispose of the stream
transformStream?.Dispose();
}
// Dispose of the stream
fileStream?.Dispose();
}
return(rebuilt);
}
/// <summary>
/// Wrap converting and updating DAT file from any format to any format
/// </summary>
/// <param name="inputPaths">List of input filenames</param>
/// <param name="basePaths">List of base filenames</param>
/// /* Normal DAT header info */
/// <param name="datHeader">All DatHeader info to be used</param>
/// /* Merging and Diffing info */
/// <param name="updateMode">Non-zero flag for diffing mode, zero otherwise</param>
/// <param name="inplace">True if the cascade-diffed files should overwrite their inputs, false otherwise</param>
/// <param name="skip">True if the first cascaded diff file should be skipped on output, false otherwise</param>
/// <param name="bare">True if the date should not be appended to the default name, false otherwise</param>
/// /* Filtering info */
/// <param name="filter">Pre-populated filter object for DAT filtering</param>
/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
/// /* Output DAT info */
/// <param name="outDir">Optional param for output directory</param>
/// <param name="clean">True to clean the game names to WoD standard, false otherwise (default)</param>
/// <param name="remUnicode">True if we should remove non-ASCII characters from output, false otherwise (default)</param>
/// <param name="descAsName">True if descriptions should be used as names, false otherwise (default)</param>
/// <param name="replaceMode">ReplaceMode representing what should be updated [only for base replacement]</param>
/// <param name="onlySame">True if descriptions should only be replaced if the game name is the same, false otherwise [only for base replacement]</param>
private static void InitUpdate(
List <string> inputPaths,
List <string> basePaths,
/* Normal DAT header info */
DatHeader datHeader,
/* Merging and Diffing info */
UpdateMode updateMode,
bool inplace,
bool skip,
bool bare,
/* Filtering info */
Filter filter,
SplitType splitType,
/* Output DAT info */
string outDir,
bool clean,
bool remUnicode,
bool descAsName,
ReplaceMode replaceMode,
bool onlySame)
{
// Normalize the extensions
datHeader.AddExtension = (String.IsNullOrWhiteSpace(datHeader.AddExtension) || datHeader.AddExtension.StartsWith(".")
? datHeader.AddExtension
: "." + datHeader.AddExtension);
datHeader.ReplaceExtension = (String.IsNullOrWhiteSpace(datHeader.ReplaceExtension) || datHeader.ReplaceExtension.StartsWith(".")
? datHeader.ReplaceExtension
: "." + datHeader.ReplaceExtension);
// If we're in a special update mode and the names aren't set, set defaults
if (updateMode != 0)
{
// Get the values that will be used
if (String.IsNullOrWhiteSpace(datHeader.Date))
{
datHeader.Date = DateTime.Now.ToString("yyyy-MM-dd");
}
if (String.IsNullOrWhiteSpace(datHeader.Name))
{
datHeader.Name = (updateMode != 0 ? "DiffDAT" : "MergeDAT")
+ (datHeader.Type == "SuperDAT" ? "-SuperDAT" : "")
+ (datHeader.DedupeRoms != DedupeType.None ? "-deduped" : "");
}
if (String.IsNullOrWhiteSpace(datHeader.Description))
{
datHeader.Description = (updateMode != 0 ? "DiffDAT" : "MergeDAT")
+ (datHeader.Type == "SuperDAT" ? "-SuperDAT" : "")
+ (datHeader.DedupeRoms != DedupeType.None ? " - deduped" : "");
if (!bare)
{
datHeader.Description += " (" + datHeader.Date + ")";
}
}
if (String.IsNullOrWhiteSpace(datHeader.Category) && updateMode != 0)
{
datHeader.Category = "DiffDAT";
}
if (String.IsNullOrWhiteSpace(datHeader.Author))
{
datHeader.Author = "SabreTools";
}
}
// If no replacement mode is set, default to Names
if (replaceMode == ReplaceMode.None)
{
replaceMode = ReplaceMode.ItemName;
}
// Populate the DatData object
DatFile userInputDat = new DatFile(datHeader);
userInputDat.DetermineUpdateType(inputPaths, basePaths, outDir, updateMode, inplace, skip, clean,
remUnicode, descAsName, filter, splitType, replaceMode, onlySame);
}
public override bool ProcessFeatures(Dictionary <string, Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
TreatAsFile asFiles = GetTreatAsFiles(features);
bool date = GetBoolean(features, AddDateValue);
bool delete = GetBoolean(features, DeleteValue);
bool inverse = GetBoolean(features, InverseValue);
bool quickScan = GetBoolean(features, QuickValue);
bool updateDat = GetBoolean(features, UpdateDatValue);
var outputFormat = GetOutputFormat(features);
// If we have the romba flag
if (Header.OutputDepot?.IsActive == true)
{
// Update TorrentGzip output
if (outputFormat == OutputFormat.TorrentGzip)
{
outputFormat = OutputFormat.TorrentGzipRomba;
}
// Update TorrentXz output
else if (outputFormat == OutputFormat.TorrentXZ)
{
outputFormat = OutputFormat.TorrentXZRomba;
}
}
// Get a list of files from the input datfiles
var datfiles = GetList(features, DatListValue);
var datfilePaths = PathTool.GetFilesOnly(datfiles);
// If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir
if (GetBoolean(features, IndividualValue))
{
foreach (ParentablePath datfile in datfilePaths)
{
DatFile datdata = DatFile.Create();
Parser.ParseInto(datdata, datfile, int.MaxValue, keep: true);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot?.Clone() as DepotInformation;
datdata.Header.OutputDepot = Header.OutputDepot?.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
// If we have the depot flag, respect it
bool success;
if (Header.InputDepot?.IsActive ?? false)
{
success = Rebuilder.RebuildDepot(datdata, Inputs, Path.Combine(OutputDir, datdata.Header.FileName), date, delete, inverse, outputFormat);
}
else
{
success = Rebuilder.RebuildGeneric(datdata, Inputs, Path.Combine(OutputDir, datdata.Header.FileName), quickScan, date, delete, inverse, outputFormat, asFiles);
}
// If we have a success and we're updating the DAT, write it out
if (success && updateDat)
{
datdata.Header.FileName = $"fixDAT_{Header.FileName}";
datdata.Header.Name = $"fixDAT_{Header.Name}";
datdata.Header.Description = $"fixDAT_{Header.Description}";
datdata.Items.ClearMarked();
Writer.Write(datdata, OutputDir);
}
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = DatFile.Create();
foreach (ParentablePath datfile in datfilePaths)
{
Parser.ParseInto(datdata, datfile, int.MaxValue, keep: true);
}
// Set depot information
datdata.Header.InputDepot = Header.InputDepot?.Clone() as DepotInformation;
datdata.Header.OutputDepot = Header.OutputDepot?.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
watch.Stop();
// If we have the depot flag, respect it
bool success;
if (Header.InputDepot?.IsActive ?? false)
{
success = Rebuilder.RebuildDepot(datdata, Inputs, OutputDir, date, delete, inverse, outputFormat);
}
else
{
success = Rebuilder.RebuildGeneric(datdata, Inputs, OutputDir, quickScan, date, delete, inverse, outputFormat, asFiles);
}
// If we have a success and we're updating the DAT, write it out
if (success && updateDat)
{
datdata.Header.FileName = $"fixDAT_{Header.FileName}";
datdata.Header.Name = $"fixDAT_{Header.Name}";
datdata.Header.Description = $"fixDAT_{Header.Description}";
datdata.Items.ClearMarked();
Writer.Write(datdata, OutputDir);
}
}
return(true);
}
static void Main(string[] args)
{
Console.WriteLine("DAT Tool by CaptainSwag101\n" +
"Version 0.0.1, built on 2019-10-22\n");
if (args.Length == 0)
{
Console.WriteLine("ERROR: No targets specified.");
return;
}
foreach (string arg in args)
{
FileInfo info = new FileInfo(arg);
if (!info.Exists)
{
Console.WriteLine($"ERROR: File \"{arg}\" does not exist, skipping.");
continue;
}
if (info.Extension.ToLowerInvariant() == ".dat")
{
// Convert DAT to CSV
DatFile dat = new DatFile();
dat.Load(info.FullName);
StringBuilder output = new StringBuilder();
// Write header
List <string> headerEntries = new List <string>();
foreach (var value in dat.ValueInfo)
{
headerEntries.Add($"{value.Name} ({value.Type})");
}
output.AppendJoin(',', headerEntries);
output.Append('\n');
// Write struct entries
List <string> structEntries = new List <string>();
foreach (var entry in dat.StructEntries)
{
StringBuilder combinedEntry = new StringBuilder();
combinedEntry.AppendJoin(',', entry);
structEntries.Add(combinedEntry.ToString());
}
output.AppendJoin('\n', structEntries);
using StreamWriter writer = new StreamWriter(info.FullName.TrimEnd(info.Extension.ToCharArray()) + ".csv", false);
writer.Write(output.ToString());
}
else if (info.Extension.ToLowerInvariant() == ".csv")
{
// Convert CSV to DAT
DatFile dat = new DatFile();
using StreamReader reader = new StreamReader(info.FullName);
// First line is header
string[] header = reader.ReadLine().Split(',');
List <(string Name, string Type)> valInfo = new List <(string Name, string Type)>();
foreach (string headerPiece in header)
{
string name = headerPiece.Split('(').First();
string type = headerPiece.Split('(').Last().TrimEnd(')');
valInfo.Add((name, type));
}
dat.ValueInfo = valInfo;
// Read struct entries
List <List <string> > entries = new List <List <string> >();
while (!reader.EndOfStream)
{
List <string> entry = new List <string>();
entry.AddRange(reader.ReadLine().Split(','));
entries.Add(entry);
}
dat.StructEntries = entries;
dat.Save(info.FullName.TrimEnd(info.Extension.ToCharArray()) + ".dat");
}
else
{
Console.WriteLine($"ERROR: Invalid file extension \"{info.Extension}\".");
continue;
}
}
}
/// <summary>
/// Wrap sorting files using an input DAT
/// </summary>
/// <param name="datfiles">Names of the DATs to compare against</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="depot">True if the input direcories are treated as romba depots, false otherwise</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
/// <param name="sevenzip">Integer representing the archive handling level for 7z</param>
/// <param name="gz">Integer representing the archive handling level for GZip</param>
/// <param name="rar">Integer representing the archive handling level for RAR</param>
/// <param name="zip">Integer representing the archive handling level for Zip</param>
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
/// <param name="chdsAsFiles">True if CHDs should be treated like regular files, false otherwise</param>
/// <param name="individual">True if DATs should be sorted individually, false if they should be done in bulk</param>
private static void InitSort(
List <string> datfiles,
List <string> inputs,
string outDir,
bool depot,
bool quickScan,
bool date,
bool delete,
bool inverse,
OutputFormat outputFormat,
bool romba,
int sevenzip,
int gz,
int rar,
int zip,
bool updateDat,
string headerToCheckAgainst,
SplitType splitType,
bool chdsAsFiles,
bool individual)
{
// Get the archive scanning level
ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(sevenzip, gz, rar, zip);
// Get a list of files from the input datfiles
datfiles = Utilities.GetOnlyFilesFromInputs(datfiles);
// If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir
if (individual)
{
foreach (string datfile in datfiles)
{
DatFile datdata = new DatFile();
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
// If we have the depot flag, respect it
if (depot)
{
datdata.RebuildDepot(inputs, Path.Combine(outDir, datdata.FileName), date, delete, inverse, outputFormat, romba,
updateDat, headerToCheckAgainst);
}
else
{
datdata.RebuildGeneric(inputs, Path.Combine(outDir, datdata.FileName), quickScan, date, delete, inverse, outputFormat, romba, asl,
updateDat, headerToCheckAgainst, chdsAsFiles);
}
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
}
watch.Stop();
// If we have the depot flag, respect it
if (depot)
{
datdata.RebuildDepot(inputs, outDir, date, delete, inverse, outputFormat, romba,
updateDat, headerToCheckAgainst);
}
else
{
datdata.RebuildGeneric(inputs, outDir, quickScan, date, delete, inverse, outputFormat, romba, asl,
updateDat, headerToCheckAgainst, chdsAsFiles);
}
}
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
public SabreXML(DatFile datFile)
: base(datFile)
{
}
void createThumbnails()
{
#region array assignment
thumbs[0] = thmb0;
thumbs[1] = thmb1;
thumbs[2] = thmb2;
thumbs[3] = thmb3;
thumbs[4] = thmb4;
thumbs[5] = thmb5;
thumbs[6] = thmb6;
thumbs[7] = thmb7;
thumbs[8] = thmb8;
thumbs[9] = thmb9;
thumbs[10] = thmb10;
thumbs[11] = thmb11;
thumbs[12] = thmb12;
thumbs[13] = thmb13;
thumbs[14] = thmb14;
thumbs[15] = thmb15;
thumbs[16] = thmb16;
thumbs[17] = thmb17;
thumbs[18] = thmb18;
thumbs[19] = thmb19;
thumbs[20] = thmb20;
thumbs[21] = thmb21;
thumbs[22] = thmb22;
thumbs[23] = thmb23;
thumbs[24] = thmb24;
thumbs[25] = thmb25;
thumbs[26] = thmb26;
thumbs[27] = thmb27;
thumbs[28] = thmb28;
thumbs[29] = thmb29;
thumbs[30] = thmb30;
thumbs[31] = thmb31;
thumbs[32] = thmb32;
thumbs[33] = thmb33;
thumbs[34] = thmb34;
thumbs[35] = thmb35;
thumbs[36] = thmb36;
thumbs[37] = thmb37;
thumbs[38] = thmb38;
thumbs[39] = thmb39;
thumbs[40] = thmb40;
thumbs[41] = thmb41;
thumbs[42] = thmb42;
thumbs[43] = thmb43;
thumbs[44] = thmb44;
thumbs[45] = thmb45;
thumbs[46] = thmb46;
thumbs[47] = thmb47;
thumbs[48] = thmb48;
thumbs[49] = thmb49;
thumbs[50] = thmb50;
thumbs[51] = thmb51;
thumbs[52] = thmb52;
thumbs[53] = thmb53;
thumbs[54] = thmb54;
thumbs[55] = thmb55;
thumbs[56] = thmb56;
thumbs[57] = thmb57;
thumbs[58] = thmb58;
thumbs[59] = thmb59;
thumbs[60] = thmb60;
thumbs[61] = thmb61;
thumbs[62] = thmb62;
thumbs[63] = thmb63;
thumbs[64] = thmb64;
thumbs[65] = thmb65;
thumbs[66] = thmb66;
thumbs[67] = thmb67;
thumbs[68] = thmb68;
thumbs[69] = thmb69;
thumbs[70] = thmb70;
thumbs[71] = thmb71;
thumbs[72] = thmb72;
thumbs[73] = thmb73;
thumbs[74] = thmb74;
thumbs[75] = thmb75;
thumbs[76] = thmb76;
thumbs[77] = thmb77;
thumbs[78] = thmb78;
thumbs[79] = thmb79;
thumbs[80] = thmb80;
thumbs[81] = thmb81;
thumbs[82] = thmb82;
thumbs[83] = thmb83;
thumbs[84] = thmb84;
thumbs[85] = thmb85;
thumbs[86] = thmb86;
thumbs[87] = thmb87;
thumbs[88] = thmb88;
thumbs[89] = thmb89;
thumbs[90] = thmb90;
thumbs[91] = thmb91;
thumbs[92] = thmb92;
thumbs[93] = thmb93;
thumbs[94] = thmb94;
thumbs[95] = thmb95;
thumbs[96] = thmb96;
thumbs[97] = thmb97;
thumbs[98] = thmb98;
thumbs[99] = thmb99;
thumbs[100] = thmb100;
thumbs[101] = thmb101;
thumbs[102] = thmb102;
#endregion
for (int i = 0; i < 103; i++)
{
thumbs[i].Tag = i;
thumbs[i].BackgroundImageLayout = ImageLayout.Zoom;
}
// TODO: also look for customs in TIE-BoP
if (_platform == MissionFile.Platform.TIE || _platform == MissionFile.Platform.XvT)
{
for (int i = 1; i <= 8; i++)
{
setThumbnail(_backdropDirectory + "PLANET" + i.ToString() + ".ACT", i - 1);
}
}
else if (_platform == MissionFile.Platform.BoP)
{
for (int i = 1; i <= 8; i++)
{
setThumbnail(_backdropDirectory.Remove(_backdropDirectory.Length - 24, 15) + "PLANET" + i.ToString() + ".ACT", i - 1);
}
for (int i = 9; i <= 12; i++)
{
setThumbnail(_backdropDirectory + "PLANET" + i.ToString() + ".ACT", i - 1);
}
setThumbnail(_backdropDirectory + "SUN1B.ACT", 12);
setThumbnail(_backdropDirectory + "SUN1C.ACT", 13);
setThumbnail(_backdropDirectory + "SUN2B.ACT", 14);
setThumbnail(_backdropDirectory + "SUN3A.ACT", 15);
setThumbnail(_backdropDirectory + "SUN4A.ACT", 16);
}
else
{
thumbs[24].Enabled = false;
_planets = new DatFile();
_planets.Groups.AutoSort = false;
setThumbnail("planet.dat", 0, 24);
_planets.Groups.Add(-1);
setThumbnail("planet.dat", 25, 35);
setThumbnail("wrapback.dat", 60, 2);
setThumbnail("dsfire.dat", 62, 1);
setThumbnail("nebula.dat", 63, 10);
setThumbnail("galaxy.dat", 73, 10);
setThumbnail("backdrop.dat", 83, 11);
setThumbnail("wrapback.dat", 94, 4);
setThumbnail("nebula.dat", 98, 5);
for (int i = 0; i < _planets.NumberOfGroups; i++)
{
if (i == 24)
{
continue;
}
thumbs[i].BackgroundImage = _planets.Groups[i].Subs[0].Image;
thumbs[i].BackColor = System.Drawing.Color.Black;
}
StreamReader sr;
try //[JB Added try block
{
sr = new StreamReader(_installDirectory + "\\RESDATA.TXT");
}
catch
{
MessageBox.Show("Could not open resource file:\n" + _installDirectory + "\\RESDATA.TXT", "Error");
return;
}
System.Collections.Generic.List <string> resdata = new System.Collections.Generic.List <string>(50);
string line = "";
while ((line = sr.ReadLine()) != null)
{
if (line != "")
{
resdata.Add(line);
}
}
for (int i = 0; i < resdata.Count - 38; i++)
{
DatFile temp = new DatFile(_installDirectory + "\\" + resdata[i]);
int index = 0;
for (int g = 0; g < temp.NumberOfGroups; g++)
{
index = _planets.Groups.GetIndex(temp.Groups[g].ID);
if (index != -1)
{
_planets.Groups[index] = temp.Groups[g];
thumbs[index].BackgroundImage = temp.Groups[g].Subs[0].Image;
}
}
}
}
}
/// <summary>
/// Replace item values from the base set represented by the current DAT
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="intDat">DatFile to replace the values in</param>
/// <param name="machineFields">List of MachineFields representing what should be updated</param>
/// <param name="datItemFields">List of DatItemFields representing what should be updated</param>
/// <param name="onlySame">True if descriptions should only be replaced if the game name is the same, false otherwise</param>
public static void BaseReplace(
DatFile datFile,
DatFile intDat,
List <MachineField> machineFields,
List <DatItemField> datItemFields,
bool onlySame)
{
InternalStopwatch watch = new InternalStopwatch($"Replacing items in '{intDat.Header.FileName}' from the base DAT");
// If we are matching based on DatItem fields of any sort
if (datItemFields.Any())
{
// For comparison's sake, we want to use CRC as the base bucketing
datFile.Items.BucketBy(ItemKey.CRC, DedupeType.Full);
intDat.Items.BucketBy(ItemKey.CRC, DedupeType.None);
// Then we do a hashwise comparison against the base DAT
Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> datItems = intDat.Items[key];
ConcurrentList <DatItem> newDatItems = new ConcurrentList <DatItem>();
foreach (DatItem datItem in datItems)
{
ConcurrentList <DatItem> dupes = datFile.Items.GetDuplicates(datItem, sorted: true);
DatItem newDatItem = datItem.Clone() as DatItem;
// Replace fields from the first duplicate, if we have one
if (dupes.Count > 0)
{
Replacer.ReplaceFields(newDatItem, dupes.First(), datItemFields);
}
newDatItems.Add(newDatItem);
}
// Now add the new list to the key
intDat.Items.Remove(key);
intDat.Items.AddRange(key, newDatItems);
});
}
// If we are matching based on Machine fields of any sort
if (machineFields.Any())
{
// For comparison's sake, we want to use Machine Name as the base bucketing
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.Full);
intDat.Items.BucketBy(ItemKey.Machine, DedupeType.None);
// Then we do a namewise comparison against the base DAT
Parallel.ForEach(intDat.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> datItems = intDat.Items[key];
ConcurrentList <DatItem> newDatItems = new ConcurrentList <DatItem>();
foreach (DatItem datItem in datItems)
{
DatItem newDatItem = datItem.Clone() as DatItem;
if (datFile.Items.ContainsKey(key) && datFile.Items[key].Count() > 0)
{
Replacer.ReplaceFields(newDatItem.Machine, datFile.Items[key][0].Machine, machineFields, onlySame);
}
newDatItems.Add(newDatItem);
}
// Now add the new list to the key
intDat.Items.Remove(key);
intDat.Items.AddRange(key, newDatItems);
});
}
watch.Stop();
}
/// <summary>
/// Populate from multiple paths while returning the invividual headers
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">Paths to DATs to parse</param>
/// <returns>List of DatHeader objects representing headers</returns>
public static List <DatHeader> PopulateUserData(DatFile datFile, List <string> inputs)
{
List <ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList();
return(PopulateUserData(datFile, paths));
}
/// <summary>
/// Wrap verifying files using an input DAT
/// </summary>
/// <param name="datfiles">Names of the DATs to compare against</param>
/// <param name="inputs">Input directories to compare against</param>
/// <param name="depot">True if the input direcories are treated as romba depots, false otherwise</param>
/// <param name="hashOnly">True if only hashes should be checked, false for full file information</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="splitType">Type of the split that should be performed (split, merged, fully merged)</param>
/// <param name="chdsAsFiles">True if CHDs should be treated like regular files, false otherwise</param>
/// <param name="individual">True if DATs should be verified individually, false if they should be done in bulk</param>
/// <param name="filter">Filter object to be passed to the DatItem level</param>
private static void InitVerify(
List <string> datfiles,
List <string> inputs,
bool depot,
bool hashOnly,
bool quickScan,
string headerToCheckAgainst,
SplitType splitType,
bool chdsAsFiles,
bool individual,
Filter filter)
{
// Get the archive scanning level
ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
// Get a list of files from the input datfiles
datfiles = Utilities.GetOnlyFilesFromInputs(datfiles);
// If we are in individual mode, process each DAT on their own
if (individual)
{
foreach (string datfile in datfiles)
{
DatFile datdata = new DatFile();
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
// If we have the depot flag, respect it
if (depot)
{
datdata.VerifyDepot(inputs, headerToCheckAgainst);
}
else
{
datdata.VerifyGeneric(inputs, hashOnly, quickScan, headerToCheckAgainst, chdsAsFiles, filter);
}
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, splitType, keep: true, useTags: true);
}
watch.Stop();
// If we have the depot flag, respect it
if (depot)
{
datdata.VerifyDepot(inputs, headerToCheckAgainst);
}
else
{
datdata.VerifyGeneric(inputs, hashOnly, quickScan, headerToCheckAgainst, chdsAsFiles, filter);
}
}
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
/// <param name="quotes">Enable quotes on read and write, false otherwise</param>
public ClrMamePro(DatFile datFile, bool quotes)
: base(datFile)
{
Quotes = quotes;
}
/// <summary>
/// Apply cleaning methods to the DatFile
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if cleaning was successful, false on error</returns>
public bool ApplyCleaning(DatFile datFile, bool throwOnError = false)
{
InternalStopwatch watch = new InternalStopwatch("Applying cleaning steps to DAT");
try
{
// Perform item-level cleaning
CleanDatItems(datFile);
// Bucket and dedupe according to the flag
if (DedupeRoms == DedupeType.Full)
{
datFile.Items.BucketBy(ItemKey.CRC, DedupeRoms);
}
else if (DedupeRoms == DedupeType.Game)
{
datFile.Items.BucketBy(ItemKey.Machine, DedupeRoms);
}
// Process description to machine name
if (DescriptionAsName == true)
{
MachineDescriptionToName(datFile);
}
// If we are removing scene dates, do that now
if (SceneDateStrip == true)
{
StripSceneDatesFromItems(datFile);
}
// Run the one rom per game logic, if required
if (OneGamePerRegion == true)
{
SetOneGamePerRegion(datFile);
}
// Run the one rom per game logic, if required
if (OneRomPerGame == true)
{
SetOneRomPerGame(datFile);
}
// Remove all marked items
datFile.Items.ClearMarked();
// We remove any blanks, if we aren't supposed to have any
if (KeepEmptyGames == false)
{
datFile.Items.ClearEmpty();
}
}
catch (Exception ex) when(!throwOnError)
{
logger.Error(ex);
return(false);
}
finally
{
watch.Stop();
}
return(true);
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
public OfflineList(DatFile datFile)
: base(datFile)
{
}
/// <summary>
/// Use game descriptions as names in the DAT, updating cloneof/romof/sampleof
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
internal void MachineDescriptionToName(DatFile datFile, bool throwOnError = false)
{
try
{
// First we want to get a mapping for all games to description
ConcurrentDictionary <string, string> mapping = new ConcurrentDictionary <string, string>();
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the key mapping doesn't exist, add it
mapping.TryAdd(item.Machine.Name, item.Machine.Description.Replace('/', '_').Replace("\"", "''").Replace(":", " -"));
}
});
// Now we loop through every item and update accordingly
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = datFile.Items[key];
ConcurrentList <DatItem> newItems = new ConcurrentList <DatItem>();
foreach (DatItem item in items)
{
// Update machine name
if (!string.IsNullOrWhiteSpace(item.Machine.Name) && mapping.ContainsKey(item.Machine.Name))
{
item.Machine.Name = mapping[item.Machine.Name];
}
// Update cloneof
if (!string.IsNullOrWhiteSpace(item.Machine.CloneOf) && mapping.ContainsKey(item.Machine.CloneOf))
{
item.Machine.CloneOf = mapping[item.Machine.CloneOf];
}
// Update romof
if (!string.IsNullOrWhiteSpace(item.Machine.RomOf) && mapping.ContainsKey(item.Machine.RomOf))
{
item.Machine.RomOf = mapping[item.Machine.RomOf];
}
// Update sampleof
if (!string.IsNullOrWhiteSpace(item.Machine.SampleOf) && mapping.ContainsKey(item.Machine.SampleOf))
{
item.Machine.SampleOf = mapping[item.Machine.SampleOf];
}
// Add the new item to the output list
newItems.Add(item);
}
// Replace the old list of roms with the new one
datFile.Items.Remove(key);
datFile.Items.AddRange(key, newItems);
});
}
catch (Exception ex) when(!throwOnError)
{
logger.Warning(ex.ToString());
}
}
/// <summary>
/// Attempt to add a file to the output if it matches
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if the file was used to rebuild, false otherwise</returns>
private static bool RebuildGenericHelper(
DatFile datFile,
string file,
string outDir,
bool quickScan,
bool date,
bool inverse,
OutputFormat outputFormat,
TreatAsFile asFiles)
{
// If we somehow have a null filename, return
if (file == null)
{
return(false);
}
// Set the deletion variables
bool usedExternally = false, usedInternally = false;
// Create an empty list of BaseFile for archive entries
List <BaseFile> entries = null;
// Get the TGZ and TXZ status for later
GZipArchive tgz = new GZipArchive(file);
XZArchive txz = new XZArchive(file);
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))
{
BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles);
// Create the correct DatItem
DatItem internalDatItem;
if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
internalDatItem = new Media(internalFileInfo);
}
else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
internalDatItem = new Disk(internalFileInfo);
}
else
{
internalDatItem = new Rom(internalFileInfo);
}
usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat);
}
// Otherwise, loop through the entries and try to match
else
{
foreach (BaseFile entry in entries)
{
DatItem internalDatItem = DatItem.Create(entry);
usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */);
}
}
return(usedExternally || usedInternally);
}
/// <summary>
/// Filter a DAT using 1G1R logic given an ordered set of regions
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
/// <remarks>
/// In the most technical sense, the way that the region list is being used does not
/// confine its values to be just regions. Since it's essentially acting like a
/// specialized version of the machine name filter, anything that is usually encapsulated
/// in parenthesis would be matched on, including disc numbers, languages, editions,
/// and anything else commonly used. Please note that, unlike other existing 1G1R
/// solutions, this does not have the ability to contain custom mappings of parent
/// to clone sets based on name, nor does it have the ability to match on the
/// Release DatItem type.
/// </remarks>
internal void SetOneGamePerRegion(DatFile datFile)
{
// If we have null region list, make it empty
if (RegionList == null)
{
RegionList = new List <string>();
}
// For sake of ease, the first thing we want to do is bucket by game
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
// Then we want to get a mapping of all machines to parents
Dictionary <string, List <string> > parents = new Dictionary <string, List <string> >();
foreach (string key in datFile.Items.Keys)
{
DatItem item = datFile.Items[key][0];
// Match on CloneOf first
if (!string.IsNullOrEmpty(item.Machine.CloneOf))
{
if (!parents.ContainsKey(item.Machine.CloneOf.ToLowerInvariant()))
{
parents.Add(item.Machine.CloneOf.ToLowerInvariant(), new List <string>());
}
parents[item.Machine.CloneOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
// Then by RomOf
else if (!string.IsNullOrEmpty(item.Machine.RomOf))
{
if (!parents.ContainsKey(item.Machine.RomOf.ToLowerInvariant()))
{
parents.Add(item.Machine.RomOf.ToLowerInvariant(), new List <string>());
}
parents[item.Machine.RomOf.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
// Otherwise, treat it as a parent
else
{
if (!parents.ContainsKey(item.Machine.Name.ToLowerInvariant()))
{
parents.Add(item.Machine.Name.ToLowerInvariant(), new List <string>());
}
parents[item.Machine.Name.ToLowerInvariant()].Add(item.Machine.Name.ToLowerInvariant());
}
}
// Once we have the full list of mappings, filter out games to keep
foreach (string key in parents.Keys)
{
// Find the first machine that matches the regions in order, if possible
string machine = default;
foreach (string region in RegionList)
{
machine = parents[key].FirstOrDefault(m => Regex.IsMatch(m, @"\(.*" + region + @".*\)", RegexOptions.IgnoreCase));
if (machine != default)
{
break;
}
}
// If we didn't get a match, use the parent
if (machine == default)
{
machine = key;
}
// Remove the key from the list
parents[key].Remove(machine);
// Remove the rest of the items from this key
parents[key].ForEach(k => datFile.Items.Remove(k));
}
// Finally, strip out the parent tags
Splitter.RemoveTagsFromChild(datFile);
}
/// <summary>
/// Process the DAT and find all matches in input files and folders assuming they're a depot
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public static bool RebuildDepot(
DatFile datFile,
List <string> inputs,
string outDir,
bool date = false,
bool delete = false,
bool inverse = false,
OutputFormat outputFormat = OutputFormat.Folder)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (datFile.Items.TotalCount == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return(false);
}
// Check that the output directory exists
outDir = outDir.Ensure(create: true);
// Now we want to get forcepack flag if it's not overridden
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None)
{
outputFormat = GetOutputFormat(datFile.Header.ForcePacking);
}
#endregion
bool success = true;
#region Rebuild from depots in order
string format = FromOutputFormat(outputFormat) ?? string.Empty;
InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}");
// Now loop through and get only directories from the input paths
List <string> directories = new List <string>();
Parallel.ForEach(inputs, Globals.ParallelOptions, input =>
{
// Add to the list if the input is a directory
if (Directory.Exists(input))
{
logger.Verbose($"Adding depot: {input}");
lock (directories)
{
directories.Add(input);
}
}
});
// If we don't have any directories, we want to exit
if (directories.Count == 0)
{
return(success);
}
// Now that we have a list of depots, we want to bucket the input DAT by SHA-1
datFile.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Then we want to loop through each of the hashes and see if we can rebuild
var keys = datFile.Items.SortedKeys.ToList();
foreach (string hash in keys)
{
// Pre-empt any issues that could arise from string length
if (hash.Length != Constants.SHA1Length)
{
continue;
}
logger.User($"Checking hash '{hash}'");
// Get the extension path for the hash
string subpath = Utilities.GetDepotPath(hash, datFile.Header.InputDepot.Depth);
// Find the first depot that includes the hash
string foundpath = null;
foreach (string directory in directories)
{
if (File.Exists(Path.Combine(directory, subpath)))
{
foundpath = Path.Combine(directory, subpath);
break;
}
}
// If we didn't find a path, then we continue
if (foundpath == null)
{
continue;
}
// If we have a path, we want to try to get the rom information
GZipArchive archive = new GZipArchive(foundpath);
BaseFile fileinfo = archive.GetTorrentGZFileInfo();
// If the file information is null, then we continue
if (fileinfo == null)
{
continue;
}
// Ensure we are sorted correctly (some other calls can change this)
datFile.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// If there are no items in the hash, we continue
if (datFile.Items[hash] == null || datFile.Items[hash].Count == 0)
{
continue;
}
// Otherwise, we rebuild that file to all locations that we need to
bool usedInternally;
if (datFile.Items[hash][0].ItemType == ItemType.Disk)
{
usedInternally = RebuildIndividualFile(datFile, new Disk(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
}
else if (datFile.Items[hash][0].ItemType == ItemType.Media)
{
usedInternally = RebuildIndividualFile(datFile, new Media(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
}
else
{
usedInternally = RebuildIndividualFile(datFile, new Rom(fileinfo), foundpath, outDir, date, inverse, outputFormat, false /* isZip */);
}
// If we are supposed to delete the depot file, do so
if (delete && usedInternally)
{
File.Delete(foundpath);
}
}
watch.Stop();
#endregion
return(success);
}
private bool LoadRomFromDat(DatFileLoader dfl, DatDir parentDir)
{
dfl.Gn();
if (dfl.Next != "(")
{
_errorReport?.Invoke(dfl.Filename, "( not found after rom, on line " + dfl.LineNumber);
return(false);
}
dfl.Gn();
if (dfl.Next.ToLower() != "name")
{
_errorReport?.Invoke(dfl.Filename, "Name not found as first object in ( ), on line " + dfl.LineNumber);
return(false);
}
DatFile dRom = new DatFile(DatFileType.UnSet)
{
Name = dfl.Gn()
};
dfl.Gn();
while (dfl.Next != ")")
{
switch (dfl.Next.ToLower())
{
case "size":
dRom.Size = VarFix.ULong(dfl.Gn());
break;
case "hash":
dfl.Gn();
break;
case "crc":
dRom.CRC = VarFix.CleanMD5SHA1(dfl.Gn(), 8);
break;
case "sha1":
dRom.SHA1 = VarFix.CleanMD5SHA1(dfl.Gn(), 40);
break;
case "md5":
dRom.MD5 = VarFix.CleanMD5SHA1(dfl.Gn(), 32);
break;
case "merge":
dRom.Merge = VarFix.String(dfl.Gn());
break;
case "flags":
string flags = VarFix.ToLower(dfl.Gn());
if (string.IsNullOrWhiteSpace(dRom.Status))
{
dRom.Status = flags;
}
break;
case "date":
dfl.Gn();
break;
case "bios":
dfl.Gn();
break;
case "region":
dfl.Gn();
break;
case "offs":
dfl.Gn();
break;
case "nodump":
dRom.Status = "nodump";
break;
case "baddump":
dRom.Status = "baddump";
break;
default:
_errorReport?.Invoke(dfl.Filename, "Error: key word '" + dfl.Next + "' not known in rom, on line " + dfl.LineNumber);
break;
}
dfl.Gn();
}
parentDir.ChildAdd(dRom);
return(true);
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
public ClrMamePro(DatFile datFile)
: base(datFile, cloneHeader: false)
{
}
private bool LoadDiskFromDat(DatFileLoader dfl, DatDir parentDir)
{
dfl.Gn();
if (dfl.Next != "(")
{
_errorReport?.Invoke(dfl.Filename, "( not found after rom, on line " + dfl.LineNumber);
return(false);
}
dfl.Gn();
if (dfl.Next.ToLower() != "name")
{
_errorReport?.Invoke(dfl.Filename, "Name not found as first object in ( ), on line " + dfl.LineNumber);
return(false);
}
DatFile dRom = new DatFile(DatFileType.UnSet)
{
Name = VarFix.String(dfl.Gn()) + ".chd",
isDisk = true
};
dfl.Gn();
while (dfl.Next != ")")
{
switch (dfl.Next.ToLower())
{
case "sha1":
dRom.SHA1 = VarFix.CleanMD5SHA1(dfl.Gn(), 40);
break;
case "md5":
dRom.MD5 = VarFix.CleanMD5SHA1(dfl.Gn(), 32);
break;
case "region":
dRom.Region = VarFix.String(dfl.Gn());
break;
case "merge":
dRom.Merge = VarFix.String(dfl.Gn());
break;
case "index":
dfl.Gn();
break;
case "flags":
dRom.Status = VarFix.ToLower(dfl.Gn());
break;
case "nodump":
dRom.Status = "nodump";
break;
default:
_errorReport?.Invoke(dfl.Filename, "Error: key word '" + dfl.Next + "' not known in rom, on line " + dfl.LineNumber);
break;
}
dfl.Gn();
}
parentDir.ChildAdd(dRom);
return(true);
}
public Race(string filename, string playerVehicleFile)
{
Race.Current = this;
Logger.Log("Starting race " + Path.GetFileName(filename));
ConfigFile = new RaceFile(filename);
foreach (string matFileName in ConfigFile.MaterialFiles)
{
MatFile matFile = new MatFile(matFileName);
ResourceCache.Add(matFile);
}
foreach (string pixFileName in ConfigFile.PixFiles)
{
PixFile pixFile = new PixFile(pixFileName);
ResourceCache.Add(pixFile);
}
if (GameVars.Emulation == EmulationMode.Demo)
{
ResourceCache.Add(new CMaterial("drkcurb.mat", 226)); //demo doesn't have this file, I guess the color is hard-coded
}
else
{
ResourceCache.Add(new MatFile("drkcurb.mat"));
}
ResourceCache.ResolveMaterials();
if (filename.Contains("TESTMAP")) //nasty hack...
{
GameVars.Scale.Y *= 0.5f;
}
DatFile modelFile = new DatFile(ConfigFile.ModelFile);
ActFile actFile = new ActFile(ConfigFile.ActorFile);
_actors = actFile.Hierarchy;
_actors.AttachModels(modelFile.Models);
_actors.ResolveTransforms(false, ConfigFile.Grooves);
if (filename.Contains("TESTMAP")) //nasty hack...
{
GameVars.Scale.Y *= 2f;
}
// link the actors and grooves
foreach (BaseGroove g in ConfigFile.Grooves)
{
g.SetActor(_actors.GetByName(g.ActorName));
}
// link the funks and materials
foreach (BaseFunk f in ConfigFile.Funks)
{
f.Resolve();
}
if (ConfigFile.SkyboxTexture != "none")
{
PixFile horizonPix = new PixFile(ConfigFile.SkyboxTexture);
_skybox = SkyboxGenerator.Generate(horizonPix.PixMaps[0].Texture, ConfigFile.SkyboxRepetitionsX - 3f, ConfigFile.DepthCueMode);
_skybox.HeightOffset = -220 + ConfigFile.SkyboxPositionY * 1.5f;
}
Physics.TrackProcessor.GenerateTrackActor(ConfigFile, _actors, out _nonCars);
Logger.Log("NonCars: " + _nonCars.Count);
GridPlacer.Reset();
List <int> opponentIds = new List <int>();
List <int> pickedNbrs = new List <int>();
for (int i = 0; i < 5; i++)
{
int index = 0;
while (true)
{
index = Engine.Random.Next(1, OpponentsFile.Instance.Opponents.Count);
if (!pickedNbrs.Contains(index))
{
pickedNbrs.Add(index);
break;
}
}
try
{
Opponents.Add(new Opponent(OpponentsFile.Instance.Opponents[index].FileName, ConfigFile.GridPosition, ConfigFile.GridDirection));
NbrOpponents++;
}
catch (Exception ex)
{
Logger.Log("Error while loading opponent " + OpponentsFile.Instance.Opponents[index].FileName + ", " + ex.Message);
}
}
foreach (CopStartPoint point in ConfigFile.CopStartPoints)
{
Opponents.Add(new Opponent(point.IsSpecialForces ? "bigapc.txt" : "apc.txt", point.Position, 0, new CopDriver()));
}
foreach (Opponent o in Opponents)
{
Drivers.Add(o.Driver);
}
OpponentController.Nodes = ConfigFile.OpponentPathNodes;
PlayerVehicle = new Vehicle(GameVars.BasePath + @"cars\" + playerVehicleFile, new PlayerDriver());
PlayerVehicle.PlaceOnGrid(ConfigFile.GridPosition, ConfigFile.GridDirection);
Drivers.Add(PlayerVehicle.Driver);
Peds = new PedestrianController(ConfigFile.Peds);
_map = new RaceMap(this);
RaceTime = new RaceTimeController();
PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.TrackId, PhysXConsts.VehicleId, ContactPairFlag.Forces | ContactPairFlag.OnStartTouch | ContactPairFlag.OnTouch);
PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.VehicleId, PhysXConsts.NonCarId, ContactPairFlag.Forces | ContactPairFlag.OnStartTouch | ContactPairFlag.OnTouch);
PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.TrackId, PhysXConsts.NonCarId, ContactPairFlag.OnTouch);
PhysX.Instance.Scene.SetActorGroupPairFlags(PhysXConsts.VehicleId, PhysXConsts.VehicleId, ContactPairFlag.Forces | ContactPairFlag.OnTouch | ContactPairFlag.OnStartTouch | ContactPairFlag.OnEndTouch);
}
/// <summary>
/// Constructor designed for casting a base DatFile
/// </summary>
/// <param name="datFile">Parent DatFile to copy from</param>
/// <param name="deprecated">True if the output uses "game", false if the output uses "machine"</param>
public Logiqx(DatFile datFile, bool deprecated)
: base(datFile)
{
_deprecated = deprecated;
}
private bool LoadDisks(DatFileLoader dfl, DatDir parentDir)
{
if (dfl.Next.ToLower() != "[disks]")
{
_errorReport?.Invoke(dfl.Filename, "Looking for [DISKS] but found " + dfl.Next + " , " + dfl.LineNumber);
return(false);
}
while (!dfl.EndOfStream())
{
string line = dfl.Gn();
if (line.Substring(0, 1) == "[")
{
return(true);
}
string[] parts = line.Split('¬');
// 1 parent name = clone of
// 2 parent description = description (from parent)
// 3 game name = name (game)
// 4 game description = description
// 5 rom name = name (rom)
// 6 rom crc = crc
// 7 rom size = size
// 8 romof name = romof
// 9 merge name = merge
string ParentName = parts[1];
string ParentDescription = parts[2];
string GameName = parts[3];
string GameDescription = parts[4];
string romName = parts[5];
string romCRC = parts[6];
string romSize = parts[7];
string romOf = parts[8];
string merge = parts[9];
int index;
DatDir dDir;
DatDir searchDir = new DatDir(DatFileType.Dir)
{
Name = GameName
};
if (parentDir.ChildNameSearch(searchDir, out index) != 0)
{
dDir = new DatDir(DatFileType.UnSet)
{
Name = GameName, DGame = new DatGame()
};
DatGame dGame = dDir.DGame;
dGame.Description = GameDescription;
if (ParentName != GameName)
{
dGame.CloneOf = ParentName;
}
parentDir.ChildAdd(dDir);
}
else
{
dDir = (DatDir)parentDir.Child(index);
// need to check everything matches
}
DatFile dRom = new DatFile(DatFileType.UnSet)
{
isDisk = true,
Name = romName,
SHA1 = VarFix.CleanMD5SHA1(romCRC, 40),
Merge = merge
};
// dRom.Size = VarFix.ULong(romSize);
// check romof=ParentName
dDir.ChildAdd(dRom);
}
return(true);
}
public void Start()
{
dispatcher = new Dispatcher();
scheduler = new Scheduler(dispatcher);
listeners = new List <Listener>();
listeners.Add(new Listener(7171, socket => new LoginConnection(this, socket)));
listeners.Add(new Listener(7172, socket => new GameConnection(this, socket)));
Clock = new Clock(12, 0);
Logger = new Logger();
Channels = new ChannelCollection();
RuleViolations = new RuleViolationCollection();
PacketsFactory = new PacketsFactory();
using (Logger.Measure("Loading items", true))
{
var otb = OtbFile.Load("data/items/items.otb");
var dat = DatFile.Load("data/items/tibia.dat");
var items = ItemsFile.Load("data/items/items.xml");
ItemFactory = new ItemFactory(otb, dat, items);
}
using (Logger.Measure("Loading monsters", true))
{
var monsters = MonsterFile.Load("data/monsters");
MonsterFactory = new MonsterFactory(monsters);
}
using (Logger.Measure("Loading npcs", true))
{
var npcs = NpcFile.Load("data/npcs");
NpcFactory = new NpcFactory(npcs);
}
using (Logger.Measure("Loading map", true))
{
var otbm = OtbmFile.Load("data/map/pholium3.otbm");
Map = new Map(ItemFactory, otbm);
}
Pathfinding = new Pathfinding(Map);
Events = new EventsCollection();
Scripts = new ScriptsCollection();
using (Logger.Measure("Loading scripts", true))
{
Scripts.Start(this);
}
dispatcher.Start();
scheduler.Start();
foreach (var listener in listeners)
{
listener.Start();
}
Logger.WriteLine("Server online");
}
private int InternalHandle(DatFile file)
{
return (int)typeof(DatFile).GetProperty("Handle", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(file, null);
}
/// <summary>
/// Output non-duplicate item diff
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static DatFile DiffNoDuplicates(DatFile datFile, List <string> inputs)
{
List <ParentablePath> paths = inputs.Select(i => new ParentablePath(i)).ToList();
return(DiffNoDuplicates(datFile, paths));
}
