/// <summary>
/// Clean a DatItem according to the cleaner
/// </summary>
/// <param name="datItem">DatItem to clean</param>
internal void CleanDatItem(DatItem datItem)
{
// If we're stripping unicode characters, strip machine name and description
if (RemoveUnicode)
{
datItem.Machine.Name = RemoveUnicodeCharacters(datItem.Machine.Name);
datItem.Machine.Description = RemoveUnicodeCharacters(datItem.Machine.Description);
datItem.SetName(RemoveUnicodeCharacters(datItem.GetName()));
}
// If we're in cleaning mode, sanitize machine name and description
if (Clean)
{
datItem.Machine.Name = CleanGameName(datItem.Machine.Name);
datItem.Machine.Description = CleanGameName(datItem.Machine.Description);
}
// If we are in single game mode, rename the machine
if (Single)
{
datItem.Machine.Name = "!";
}
// If we are in NTFS trim mode, trim the item name
if (Trim && datItem.GetName() != null)
{
// Windows max name length is 260
int usableLength = 260 - datItem.Machine.Name.Length - (Root?.Length ?? 0);
if (datItem.GetName().Length > usableLength)
{
string ext = Path.GetExtension(datItem.GetName());
datItem.SetName(datItem.GetName().Substring(0, usableLength - ext.Length) + ext);
}
}
}
/// <summary>
/// Set internal names to match One Rom Per Game (ORPG) logic
/// </summary>
/// <param name="datItem">DatItem to run logic on</param>
internal void SetOneRomPerGame(DatItem datItem)
{
if (datItem.GetName() == null)
{
return;
}
string[] splitname = datItem.GetName().Split('.');
datItem.Machine.Name += $"/{string.Join(".", splitname.Take(splitname.Length > 1 ? splitname.Length - 1 : 1))}";
datItem.SetName(Path.GetFileName(datItem.GetName()));
}
/// <summary>
/// Read set information
/// </summary>
/// <param name="cmpr">ClrMameProReader to use to parse the header</param>
/// <param name="resource">True if the item is a resource (bios), false otherwise</param>
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
private void ReadSet(
ClrMameProReader cmpr,
bool resource,
bool statsOnly,
// Standard Dat parsing
string filename,
int indexId)
{
// Prepare all internal variables
bool containsItems = false;
Machine machine = new Machine()
{
MachineType = (resource ? MachineType.Bios : MachineType.NULL),
};
// If there's no subtree to the header, skip it
if (cmpr == null || cmpr.EndOfStream)
{
return;
}
// While we don't hit an end element or end of stream
while (!cmpr.EndOfStream)
{
cmpr.ReadNextLine();
// Ignore comments and nothingness
if (cmpr.RowType == CmpRowType.None || cmpr.RowType == CmpRowType.Comment)
{
continue;
}
// If we reached the end of a section, break
if (cmpr.RowType == CmpRowType.EndTopLevel)
{
break;
}
// Handle any standalone items
if (cmpr.RowType == CmpRowType.Standalone && cmpr.Standalone != null)
{
string itemKey = cmpr.Standalone?.Key.ToLowerInvariant();
string itemVal = cmpr.Standalone?.Value;
switch (itemKey)
{
case "name":
machine.Name = itemVal;
break;
case "description":
machine.Description = itemVal;
break;
case "year":
machine.Year = itemVal;
break;
case "manufacturer":
machine.Manufacturer = itemVal;
break;
case "category":
machine.Category = itemVal;
break;
case "cloneof":
machine.CloneOf = itemVal;
break;
case "romof":
machine.RomOf = itemVal;
break;
case "sampleof":
machine.SampleOf = itemVal;
break;
}
}
// Handle any internal items
else if (cmpr.RowType == CmpRowType.Internal &&
!string.IsNullOrWhiteSpace(cmpr.InternalName) &&
cmpr.Internal != null)
{
containsItems = true;
string itemKey = cmpr.InternalName;
// Create the proper DatItem based on the type
ItemType itemType = itemKey.AsItemType() ?? ItemType.Rom;
DatItem item = DatItem.Create(itemType);
// Then populate it with information
item.CopyMachineInformation(machine);
item.Source.Index = indexId;
item.Source.Name = filename;
// Loop through all of the attributes
foreach (var kvp in cmpr.Internal)
{
string attrKey = kvp.Key;
string attrVal = kvp.Value;
switch (attrKey)
{
//If the item is empty, we automatically skip it because it's a fluke
case "":
continue;
// Regular attributes
case "name":
item.SetName(attrVal);
break;
case "size":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).Size = Utilities.CleanLong(attrVal);
}
break;
case "crc":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).CRC = attrVal;
}
break;
case "md5":
if (item.ItemType == ItemType.Disk)
{
(item as Disk).MD5 = attrVal;
}
else if (item.ItemType == ItemType.Media)
{
(item as Media).MD5 = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).MD5 = attrVal;
}
break;
case "sha1":
if (item.ItemType == ItemType.Disk)
{
(item as Disk).SHA1 = attrVal;
}
else if (item.ItemType == ItemType.Media)
{
(item as Media).SHA1 = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA1 = attrVal;
}
break;
case "sha256":
if (item.ItemType == ItemType.Media)
{
(item as Media).SHA256 = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA256 = attrVal;
}
break;
case "sha384":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA384 = attrVal;
}
break;
case "sha512":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA512 = attrVal;
}
break;
case "spamsum":
if (item.ItemType == ItemType.Media)
{
(item as Media).SpamSum = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).SpamSum = attrVal;
}
break;
case "status":
ItemStatus tempFlagStatus = attrVal.AsItemStatus();
if (item.ItemType == ItemType.Disk)
{
(item as Disk).ItemStatus = tempFlagStatus;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).ItemStatus = tempFlagStatus;
}
break;
case "date":
if (item.ItemType == ItemType.Release)
{
(item as Release).Date = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).Date = attrVal;
}
break;
case "default":
if (item.ItemType == ItemType.BiosSet)
{
(item as BiosSet).Default = attrVal.AsYesNo();
}
else if (item.ItemType == ItemType.Release)
{
(item as Release).Default = attrVal.AsYesNo();
}
break;
case "description":
if (item.ItemType == ItemType.BiosSet)
{
(item as BiosSet).Description = attrVal;
}
break;
case "region":
if (item.ItemType == ItemType.Release)
{
(item as Release).Region = attrVal;
}
break;
case "language":
if (item.ItemType == ItemType.Release)
{
(item as Release).Language = attrVal;
}
break;
}
}
// Now process and add the rom
ParseAddHelper(item, statsOnly);
}
}
// If no items were found for this machine, add a Blank placeholder
if (!containsItems)
{
Blank blank = new Blank()
{
Source = new Source
{
Index = indexId,
Name = filename,
},
};
blank.CopyMachineInformation(machine);
// Now process and add the rom
ParseAddHelper(blank, statsOnly);
}
}
/// <summary>
/// Remove fields with given values
/// </summary>
/// <param name="datItem">DatItem to remove fields from</param>
public void RemoveFields(DatItem datItem)
{
if (datItem == null)
{
return;
}
#region Common
if (MachineFields != null && MachineFields.Any() && datItem.Machine != null)
{
RemoveFields(datItem.Machine);
}
if (DatItemFields == null || !DatItemFields.Any())
{
return;
}
if (DatItemFields.Contains(DatItemField.Name))
{
datItem.SetName(null);
}
#endregion
#region Item-Specific
if (datItem is Adjuster)
{
RemoveFields(datItem as Adjuster);
}
else if (datItem is Analog)
{
RemoveFields(datItem as Analog);
}
else if (datItem is BiosSet)
{
RemoveFields(datItem as BiosSet);
}
else if (datItem is Chip)
{
RemoveFields(datItem as Chip);
}
else if (datItem is Condition)
{
RemoveFields(datItem as Condition);
}
else if (datItem is Configuration)
{
RemoveFields(datItem as Configuration);
}
else if (datItem is Control)
{
RemoveFields(datItem as Control);
}
else if (datItem is DataArea)
{
RemoveFields(datItem as DataArea);
}
else if (datItem is Device)
{
RemoveFields(datItem as Device);
}
else if (datItem is DipSwitch)
{
RemoveFields(datItem as DipSwitch);
}
else if (datItem is Disk)
{
RemoveFields(datItem as Disk);
}
else if (datItem is DiskArea)
{
RemoveFields(datItem as DiskArea);
}
else if (datItem is Display)
{
RemoveFields(datItem as Display);
}
else if (datItem is Driver)
{
RemoveFields(datItem as Driver);
}
else if (datItem is Extension)
{
RemoveFields(datItem as Extension);
}
else if (datItem is Feature)
{
RemoveFields(datItem as Feature);
}
else if (datItem is Info)
{
RemoveFields(datItem as Info);
}
else if (datItem is Input)
{
RemoveFields(datItem as Input);
}
else if (datItem is Instance)
{
RemoveFields(datItem as Instance);
}
else if (datItem is Location)
{
RemoveFields(datItem as Location);
}
else if (datItem is Media)
{
RemoveFields(datItem as Media);
}
else if (datItem is Part)
{
RemoveFields(datItem as Part);
}
else if (datItem is PartFeature)
{
RemoveFields(datItem as PartFeature);
}
else if (datItem is Port)
{
RemoveFields(datItem as Port);
}
else if (datItem is RamOption)
{
RemoveFields(datItem as RamOption);
}
else if (datItem is Release)
{
RemoveFields(datItem as Release);
}
else if (datItem is Rom)
{
RemoveFields(datItem as Rom);
}
else if (datItem is Setting)
{
RemoveFields(datItem as Setting);
}
else if (datItem is SharedFeature)
{
RemoveFields(datItem as SharedFeature);
}
else if (datItem is Slot)
{
RemoveFields(datItem as Slot);
}
else if (datItem is SlotOption)
{
RemoveFields(datItem as SlotOption);
}
else if (datItem is SoftwareList)
{
RemoveFields(datItem as SoftwareList);
}
else if (datItem is Sound)
{
RemoveFields(datItem as Sound);
}
#endregion
}
/// <summary>
/// Find duplicates and rebuild individual files to output
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
/// <returns>True if the file was able to be rebuilt, false otherwise</returns>
private static bool RebuildIndividualFile(
DatFile datFile,
DatItem datItem,
string file,
string outDir,
bool date,
bool inverse,
OutputFormat outputFormat,
bool?isZip = null)
{
// Set the initial output value
bool rebuilt = false;
// If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) &&
!(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) &&
!(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
outputFormat = OutputFormat.Folder;
}
// If we have a Disk or Media, change it into a Rom for later use
if (datItem.ItemType == ItemType.Disk)
{
datItem = (datItem as Disk).ConvertToRom();
}
else if (datItem.ItemType == ItemType.Media)
{
datItem = (datItem as Media).ConvertToRom();
}
// Prepopluate a key string
string crc = (datItem as Rom).CRC ?? string.Empty;
// Try to get the stream for the file
if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
{
return(false);
}
// If either we have duplicates or we're filtering
if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes))
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
// If we have a very specific TXZ->TXZ case, just copy it accordingly
if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Special case for partial packing mode
bool shouldCheck = false;
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
{
shouldCheck = true;
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false);
}
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// If we should check for the items in the machine
if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
{
outputFormat = OutputFormat.Folder;
}
else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
{
outputFormat = OutputFormat.ParentFolder;
}
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
}
// Close the input stream
fileStream?.Dispose();
}
// Now we want to take care of headers, if applicable
if (datFile.Header.HeaderSkipper != null)
{
// Check to see if we have a matching header first
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
MemoryStream transformStream = new MemoryStream();
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
{
logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// Create a headered item to use as well
datItem.CopyMachineInformation(item);
datItem.SetName($"{datItem.GetName()}_{crc}");
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
bool eitherSuccess = false;
eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
}
}
}
// Dispose of the stream
transformStream?.Dispose();
}
// Dispose of the stream
fileStream?.Dispose();
}
return(rebuilt);
}