/// <summary>
/// Clean a DatItem according to the cleaner
/// </summary>
/// <param name="datItem">DatItem to clean</param>
internal void CleanDatItem(DatItem datItem)
{
// If we're stripping unicode characters, strip machine name and description
if (RemoveUnicode)
{
datItem.Machine.Name = RemoveUnicodeCharacters(datItem.Machine.Name);
datItem.Machine.Description = RemoveUnicodeCharacters(datItem.Machine.Description);
datItem.SetName(RemoveUnicodeCharacters(datItem.GetName()));
}
// If we're in cleaning mode, sanitize machine name and description
if (Clean)
{
datItem.Machine.Name = CleanGameName(datItem.Machine.Name);
datItem.Machine.Description = CleanGameName(datItem.Machine.Description);
}
// If we are in single game mode, rename the machine
if (Single)
{
datItem.Machine.Name = "!";
}
// If we are in NTFS trim mode, trim the item name
if (Trim && datItem.GetName() != null)
{
// Windows max name length is 260
int usableLength = 260 - datItem.Machine.Name.Length - (Root?.Length ?? 0);
if (datItem.GetName().Length > usableLength)
{
string ext = Path.GetExtension(datItem.GetName());
datItem.SetName(datItem.GetName().Substring(0, usableLength - ext.Length) + ext);
}
}
}
/// <summary>
/// Set proper Game and Rom names from user inputs
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="datItem">DatItem representing the input file</param>
/// <param name="item">Item name to use</param>
/// <param name="parent">Parent name to use</param>
/// <param name="basepath">Base path to use</param>
private static void SetDatItemInfo(DatFile datFile, DatItem datItem, string item, string parent, string basepath)
{
// Get the data to be added as game and item names
string machineName, itemName;
// If the parent is blank, then we have a non-archive file
if (string.IsNullOrWhiteSpace(parent))
{
// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
if (datFile.Header.Type == "SuperDAT")
{
machineName = Path.GetDirectoryName(item.Remove(0, basepath.Length));
itemName = Path.GetFileName(item);
}
// Otherwise, we want just the top level folder as the game, and the file as everything else
else
{
machineName = item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0];
itemName = item.Remove(0, (Path.Combine(basepath, machineName).Length));
}
}
// Otherwise, we assume that we have an archive
else
{
// If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?)
if (datFile.Header.Type == "SuperDAT")
{
machineName = parent;
itemName = datItem.GetName();
}
// Otherwise, we want the archive name as the game, and the file as everything else
else
{
machineName = parent;
itemName = datItem.GetName();
}
}
// Sanitize the names
machineName = machineName.Trim(Path.DirectorySeparatorChar);
itemName = itemName?.Trim(Path.DirectorySeparatorChar) ?? string.Empty;
if (!string.IsNullOrWhiteSpace(machineName) && string.IsNullOrWhiteSpace(itemName))
{
itemName = machineName;
machineName = "Default";
}
// Update machine information
datItem.Machine.Name = machineName;
datItem.Machine.Description = machineName;
// If we have a Disk, then the ".chd" extension needs to be removed
if (datItem.ItemType == ItemType.Disk && itemName.EndsWith(".chd"))
{
itemName = itemName[0..^ 4];
/// <summary>
/// Set internal names to match One Rom Per Game (ORPG) logic
/// </summary>
/// <param name="datItem">DatItem to run logic on</param>
internal void SetOneRomPerGame(DatItem datItem)
{
if (datItem.GetName() == null)
{
return;
}
string[] splitname = datItem.GetName().Split('.');
datItem.Machine.Name += $"/{string.Join(".", splitname.Take(splitname.Length > 1 ? splitname.Length - 1 : 1))}";
datItem.SetName(Path.GetFileName(datItem.GetName()));
}
/// <summary>
/// Get the Stream related to a file
/// </summary>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="isZip">Non-null if the input file is an archive</param>
/// <param name="stream">Output stream representing the opened file</param>
/// <returns>True if the stream opening succeeded, false otherwise</returns>
private static bool GetFileStream(DatItem datItem, string file, bool?isZip, out Stream stream)
{
// Get a generic stream for the file
stream = null;
// If we have a zipfile, extract the stream to memory
if (isZip != null)
{
BaseArchive archive = BaseArchive.Create(file);
if (archive != null)
{
(stream, _) = archive.CopyToStream(datItem.GetName() ?? datItem.ItemType.ToString());
}
}
// Otherwise, just open the filestream
else
{
stream = File.OpenRead(file);
}
// If the stream is null, then continue
if (stream == null)
{
return(false);
}
// Seek to the beginning of the stream
if (stream.CanSeek)
{
stream.Seek(0, SeekOrigin.Begin);
}
return(true);
}
/// <summary>
/// Write out DatItem using the supplied StreamWriter
/// </summary>
/// <param name="xtw">XmlTextWriter to output to</param>
/// <param name="datItem">DatItem object to be output</param>
/// <returns>True if the data was written, false on error</returns>
private void WriteDatItem(XmlTextWriter xtw, DatItem datItem)
{
// Pre-process the item name
ProcessItemName(datItem, true);
// Build the state
xtw.WriteStartElement("game");
xtw.WriteElementString("imageNumber", "1");
xtw.WriteElementString("releaseNumber", "1");
xtw.WriteRequiredElementString("title", datItem.GetName() ?? string.Empty);
xtw.WriteElementString("saveType", "None");
if (datItem.ItemType == ItemType.Rom)
{
var rom = datItem as Rom;
xtw.WriteRequiredElementString("romSize", rom.Size?.ToString());
}
xtw.WriteRequiredElementString("publisher", datItem.Machine.Publisher);
xtw.WriteElementString("location", "0");
xtw.WriteElementString("sourceRom", "None");
xtw.WriteElementString("language", "0");
if (datItem.ItemType == ItemType.Rom)
{
var rom = datItem as Rom;
string tempext = "." + rom.Name.GetNormalizedExtension();
xtw.WriteStartElement("files");
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
xtw.WriteStartElement("romCRC");
xtw.WriteRequiredAttributeString("extension", tempext);
xtw.WriteString(rom.CRC?.ToUpperInvariant());
xtw.WriteEndElement();
}
// End files
xtw.WriteEndElement();
}
xtw.WriteElementString("im1CRC", "00000000");
xtw.WriteElementString("im2CRC", "00000000");
xtw.WriteRequiredElementString("comment", datItem.Machine.Comment);
xtw.WriteRequiredElementString("duplicateID", datItem.Machine.CloneOf);
// End game
xtw.WriteEndElement();
xtw.Flush();
}
/// <summary>
/// Write out DatItem using the supplied StreamWriter
/// </summary>
/// <param name="sw">StreamWriter to output to</param>
/// <param name="datItem">DatItem object to be output</param>
/// <param name="lastgame">The name of the last game to be output</param>
private void WriteDatItem(StreamWriter sw, DatItem datItem, string lastgame)
{
// Process the item name
ProcessItemName(datItem, false, forceRomName: false);
// Romba mode automatically uses item name
if (Header.OutputDepot?.IsActive == true || Header.UseRomName)
{
sw.Write($"{datItem.GetName() ?? string.Empty}\n");
}
else if (!Header.UseRomName && datItem.Machine.Name != lastgame)
{
sw.Write($"{datItem.Machine.Name}\n");
}
sw.Flush();
}
/// <summary>
/// Use romof tags to add roms to the children
/// </summary>
/// <param name="datFile">Current DatFile object to run operations on</param>
internal static void AddRomsFromBios(DatFile datFile)
{
List <string> games = datFile.Items.Keys.OrderBy(g => g).ToList();
foreach (string game in games)
{
// If the game has no items in it, we want to continue
if (datFile.Items[game].Count == 0)
{
continue;
}
// Determine if the game has a parent or not
string parent = null;
if (!string.IsNullOrWhiteSpace(datFile.Items[game][0].Machine.RomOf))
{
parent = datFile.Items[game][0].Machine.RomOf;
}
// If the parent doesnt exist, we want to continue
if (string.IsNullOrWhiteSpace(parent))
{
continue;
}
// If the parent doesn't have any items, we want to continue
if (datFile.Items[parent].Count == 0)
{
continue;
}
// If the parent exists and has items, we copy the items from the parent to the current game
DatItem copyFrom = datFile.Items[game][0];
ConcurrentList <DatItem> parentItems = datFile.Items[parent];
foreach (DatItem item in parentItems)
{
DatItem datItem = (DatItem)item.Clone();
datItem.CopyMachineInformation(copyFrom);
if (datFile.Items[game].Where(i => i.GetName() == datItem.GetName()).Count() == 0 && !datFile.Items[game].Contains(datItem))
{
datFile.Items.Add(game, datItem);
}
}
}
}
/// <summary>
/// Process a single file as a file (with found Rom data)
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="item">Rom data to be used to write to file</param>
/// <param name="basepath">Path the represents the parent directory</param>
/// <param name="parent">Parent game to be used</param>
private static void ProcessFileHelper(DatFile datFile, string item, DatItem datItem, string basepath, string parent)
{
// If we didn't get an accepted parsed type somehow, cancel out
List <ItemType> parsed = new List <ItemType> {
ItemType.Disk, ItemType.Media, ItemType.Rom
};
if (!parsed.Contains(datItem.ItemType))
{
return;
}
try
{
// If the basepath doesn't end with a directory separator, add it
if (!basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
{
basepath += Path.DirectorySeparatorChar.ToString();
}
// Make sure we have the full item path
item = Path.GetFullPath(item);
// Process the item to sanitize names based on input
SetDatItemInfo(datFile, datItem, item, parent, basepath);
// Add the file information to the DAT
string key = datItem.GetKey(ItemKey.CRC);
datFile.Items.Add(key, datItem);
logger.Verbose($"File added: {datItem.GetName() ?? string.Empty}");
}
catch (IOException ex)
{
logger.Error(ex);
return;
}
}
/// <summary>
/// Rebuild from TorrentXz to TorrentXz
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TXZ, null otherwise</param>
/// <returns>True if rebuilt properly, false otherwise</returns>
private static bool RebuildTorrentXz(DatFile datFile, DatItem datItem, string file, string outDir, OutputFormat outputFormat, bool?isZip)
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
XZArchive txz = new XZArchive(file);
BaseFile txzRom = txz.GetTorrentXZFileInfo();
if (isZip == false && txzRom != null && (outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
logger.User($"Matches found for '{Path.GetFileName(datItem.GetName() ?? string.Empty)}', rebuilding accordingly...");
// Get the proper output path
string sha1 = (datItem as Rom).SHA1 ?? string.Empty;
if (outputFormat == OutputFormat.TorrentXZRomba)
{
outDir = Path.Combine(outDir, Utilities.GetDepotPath(sha1, datFile.Header.OutputDepot.Depth)).Replace(".gz", ".xz");
}
else
{
outDir = Path.Combine(outDir, sha1 + ".xz");
}
// Make sure the output folder is created
Directory.CreateDirectory(Path.GetDirectoryName(outDir));
// Now copy the file over
try
{
File.Copy(file, outDir);
return(true);
}
catch
{
return(false);
}
}
return(false);
}
/// <summary>
/// Find duplicates and rebuild individual files to output
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
/// <returns>True if the file was able to be rebuilt, false otherwise</returns>
private static bool RebuildIndividualFile(
DatFile datFile,
DatItem datItem,
string file,
string outDir,
bool date,
bool inverse,
OutputFormat outputFormat,
bool?isZip = null)
{
// Set the initial output value
bool rebuilt = false;
// If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) &&
!(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) &&
!(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
outputFormat = OutputFormat.Folder;
}
// If we have a Disk or Media, change it into a Rom for later use
if (datItem.ItemType == ItemType.Disk)
{
datItem = (datItem as Disk).ConvertToRom();
}
else if (datItem.ItemType == ItemType.Media)
{
datItem = (datItem as Media).ConvertToRom();
}
// Prepopluate a key string
string crc = (datItem as Rom).CRC ?? string.Empty;
// Try to get the stream for the file
if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
{
return(false);
}
// If either we have duplicates or we're filtering
if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes))
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
// If we have a very specific TXZ->TXZ case, just copy it accordingly
if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Special case for partial packing mode
bool shouldCheck = false;
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
{
shouldCheck = true;
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false);
}
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// If we should check for the items in the machine
if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
{
outputFormat = OutputFormat.Folder;
}
else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
{
outputFormat = OutputFormat.ParentFolder;
}
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
}
// Close the input stream
fileStream?.Dispose();
}
// Now we want to take care of headers, if applicable
if (datFile.Header.HeaderSkipper != null)
{
// Check to see if we have a matching header first
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
MemoryStream transformStream = new MemoryStream();
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
{
logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// Create a headered item to use as well
datItem.CopyMachineInformation(item);
datItem.SetName($"{datItem.GetName()}_{crc}");
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
bool eitherSuccess = false;
eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
}
}
}
// Dispose of the stream
transformStream?.Dispose();
}
// Dispose of the stream
fileStream?.Dispose();
}
return(rebuilt);
}
