public void CreateBaseFileTest(FileType fileType, ItemType?expected)
{
var baseFile = CreateBaseFile(fileType);
var actual = DatItem.Create(baseFile);
Assert.Equal(expected, actual?.ItemType);
}
/// <inheritdoc/>
public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
{
// Open a file reader
Encoding enc = filename.GetEncoding();
SeparatedValueReader svr = new SeparatedValueReader(File.OpenRead(filename), enc)
{
Header = true,
Quotes = true,
Separator = _delim,
VerifyFieldCount = true,
};
// If we're somehow at the end of the stream already, we can't do anything
if (svr.EndOfStream)
{
return;
}
// Read in the header
svr.ReadHeader();
// Loop through all of the data lines
while (!svr.EndOfStream)
{
try
{
// Get the current line, split and parse
svr.ReadNextLine();
// Create mapping dictionaries
Setter setter = new Setter();
setter.PopulateSettersFromList(svr.HeaderValues, svr.Line);
// Set DatHeader fields
DatHeader datHeader = new DatHeader();
setter.SetFields(datHeader);
Header.ConditionalCopy(datHeader);
// Set Machine and DatItem fields
if (setter.DatItemMappings.ContainsKey(DatItemField.Type))
{
DatItem datItem = DatItem.Create(setter.DatItemMappings[DatItemField.Type].AsItemType());
setter.SetFields(datItem);
datItem.Machine = new Machine();
setter.SetFields(datItem.Machine);
datItem.Source = new Source(indexId, filename);
ParseAddHelper(datItem, statsOnly);
}
}
catch (Exception ex) when(!throwOnError)
{
string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'";
logger.Error(ex, message);
}
}
svr.Dispose();
}
/// <summary>
/// Process a single file as a file
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles)
{
logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles);
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);
}
/// <summary>
/// Process a single file as an archive
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="extracted">List of BaseFiles representing the internal files</param>
private static void ProcessArchive(DatFile datFile, string item, string basePath, List <BaseFile> extracted)
{
// Get the parent path for all items
string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item);
// First take care of the found items
Parallel.ForEach(extracted, Globals.ParallelOptions, baseFile =>
{
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(datFile, item, datItem, basePath, parent);
});
}
/// <summary>
/// Parse a character-separated value DAT and return all found games and roms within
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
protected override void ParseFile(string filename, int indexId, bool keep, bool throwOnError = false)
{
// Open a file reader
Encoding enc = FileExtensions.GetEncoding(filename);
SeparatedValueReader svr = new SeparatedValueReader(FileExtensions.TryOpenRead(filename), enc)
{
Header = true,
Quotes = true,
Separator = _delim,
VerifyFieldCount = true,
};
// If we're somehow at the end of the stream already, we can't do anything
if (svr.EndOfStream)
{
return;
}
// Read in the header
svr.ReadHeader();
// Loop through all of the data lines
while (!svr.EndOfStream)
{
try
{
// Get the current line, split and parse
svr.ReadNextLine();
// Create mapping dictionary
var mappings = new Dictionary <Field, string>();
// Now we loop through and get values for everything
for (int i = 0; i < svr.HeaderValues.Count; i++)
{
Field key = svr.HeaderValues[i].AsField();
string value = svr.Line[i];
mappings[key] = value;
}
// Set DatHeader fields
DatHeader header = new DatHeader();
header.SetFields(mappings);
Header.ConditionalCopy(header);
// Set Machine and DatItem fields
if (mappings.ContainsKey(Field.DatItem_Type))
{
DatItem datItem = DatItem.Create(mappings[Field.DatItem_Type].AsItemType());
datItem.SetFields(mappings);
datItem.Source = new Source(indexId, filename);
ParseAddHelper(datItem);
}
}
catch (Exception ex)
{
string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'";
logger.Error(ex, message);
if (throwOnError)
{
svr.Dispose();
throw new Exception(message, ex);
}
}
}
svr.Dispose();
}
/// <summary>
/// Read set information
/// </summary>
/// <param name="cmpr">ClrMameProReader to use to parse the header</param>
/// <param name="resource">True if the item is a resource (bios), false otherwise</param>
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
private void ReadSet(
ClrMameProReader cmpr,
bool resource,
bool statsOnly,
// Standard Dat parsing
string filename,
int indexId)
{
// Prepare all internal variables
bool containsItems = false;
Machine machine = new Machine()
{
MachineType = (resource ? MachineType.Bios : MachineType.NULL),
};
// If there's no subtree to the header, skip it
if (cmpr == null || cmpr.EndOfStream)
{
return;
}
// While we don't hit an end element or end of stream
while (!cmpr.EndOfStream)
{
cmpr.ReadNextLine();
// Ignore comments and nothingness
if (cmpr.RowType == CmpRowType.None || cmpr.RowType == CmpRowType.Comment)
{
continue;
}
// If we reached the end of a section, break
if (cmpr.RowType == CmpRowType.EndTopLevel)
{
break;
}
// Handle any standalone items
if (cmpr.RowType == CmpRowType.Standalone && cmpr.Standalone != null)
{
string itemKey = cmpr.Standalone?.Key.ToLowerInvariant();
string itemVal = cmpr.Standalone?.Value;
switch (itemKey)
{
case "name":
machine.Name = itemVal;
break;
case "description":
machine.Description = itemVal;
break;
case "year":
machine.Year = itemVal;
break;
case "manufacturer":
machine.Manufacturer = itemVal;
break;
case "category":
machine.Category = itemVal;
break;
case "cloneof":
machine.CloneOf = itemVal;
break;
case "romof":
machine.RomOf = itemVal;
break;
case "sampleof":
machine.SampleOf = itemVal;
break;
}
}
// Handle any internal items
else if (cmpr.RowType == CmpRowType.Internal &&
!string.IsNullOrWhiteSpace(cmpr.InternalName) &&
cmpr.Internal != null)
{
containsItems = true;
string itemKey = cmpr.InternalName;
// Create the proper DatItem based on the type
ItemType itemType = itemKey.AsItemType() ?? ItemType.Rom;
DatItem item = DatItem.Create(itemType);
// Then populate it with information
item.CopyMachineInformation(machine);
item.Source.Index = indexId;
item.Source.Name = filename;
// Loop through all of the attributes
foreach (var kvp in cmpr.Internal)
{
string attrKey = kvp.Key;
string attrVal = kvp.Value;
switch (attrKey)
{
//If the item is empty, we automatically skip it because it's a fluke
case "":
continue;
// Regular attributes
case "name":
item.SetName(attrVal);
break;
case "size":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).Size = Utilities.CleanLong(attrVal);
}
break;
case "crc":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).CRC = attrVal;
}
break;
case "md5":
if (item.ItemType == ItemType.Disk)
{
(item as Disk).MD5 = attrVal;
}
else if (item.ItemType == ItemType.Media)
{
(item as Media).MD5 = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).MD5 = attrVal;
}
break;
case "sha1":
if (item.ItemType == ItemType.Disk)
{
(item as Disk).SHA1 = attrVal;
}
else if (item.ItemType == ItemType.Media)
{
(item as Media).SHA1 = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA1 = attrVal;
}
break;
case "sha256":
if (item.ItemType == ItemType.Media)
{
(item as Media).SHA256 = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA256 = attrVal;
}
break;
case "sha384":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA384 = attrVal;
}
break;
case "sha512":
if (item.ItemType == ItemType.Rom)
{
(item as Rom).SHA512 = attrVal;
}
break;
case "spamsum":
if (item.ItemType == ItemType.Media)
{
(item as Media).SpamSum = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).SpamSum = attrVal;
}
break;
case "status":
ItemStatus tempFlagStatus = attrVal.AsItemStatus();
if (item.ItemType == ItemType.Disk)
{
(item as Disk).ItemStatus = tempFlagStatus;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).ItemStatus = tempFlagStatus;
}
break;
case "date":
if (item.ItemType == ItemType.Release)
{
(item as Release).Date = attrVal;
}
else if (item.ItemType == ItemType.Rom)
{
(item as Rom).Date = attrVal;
}
break;
case "default":
if (item.ItemType == ItemType.BiosSet)
{
(item as BiosSet).Default = attrVal.AsYesNo();
}
else if (item.ItemType == ItemType.Release)
{
(item as Release).Default = attrVal.AsYesNo();
}
break;
case "description":
if (item.ItemType == ItemType.BiosSet)
{
(item as BiosSet).Description = attrVal;
}
break;
case "region":
if (item.ItemType == ItemType.Release)
{
(item as Release).Region = attrVal;
}
break;
case "language":
if (item.ItemType == ItemType.Release)
{
(item as Release).Language = attrVal;
}
break;
}
}
// Now process and add the rom
ParseAddHelper(item, statsOnly);
}
}
// If no items were found for this machine, add a Blank placeholder
if (!containsItems)
{
Blank blank = new Blank()
{
Source = new Source
{
Index = indexId,
Name = filename,
},
};
blank.CopyMachineInformation(machine);
// Now process and add the rom
ParseAddHelper(blank, statsOnly);
}
}
/// <summary>
/// Read set information
/// </summary>
/// <param name="cmpr">ClrMameProReader to use to parse the header</param>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
private void ReadGame(ClrMameProReader cmpr, string filename, int indexId)
{
// Prepare all internal variables
bool containsItems = false;
Machine machine = new Machine()
{
MachineType = MachineType.NULL,
};
// If there's no subtree to the header, skip it
if (cmpr == null || cmpr.EndOfStream)
{
return;
}
// While we don't hit an end element or end of stream
while (!cmpr.EndOfStream)
{
cmpr.ReadNextLine();
// Ignore comments and nothingness
if (cmpr.RowType == CmpRowType.None || cmpr.RowType == CmpRowType.Comment)
{
continue;
}
// If we reached the end of a section, break
if (cmpr.RowType == CmpRowType.EndTopLevel)
{
break;
}
// Handle any standalone items
if (cmpr.RowType == CmpRowType.Standalone && cmpr.Standalone != null)
{
string itemKey = cmpr.Standalone?.Key.ToLowerInvariant();
string itemVal = cmpr.Standalone?.Value;
switch (itemKey)
{
case "name":
machine.Name = (itemVal.ToLowerInvariant().EndsWith(".zip") ? itemVal.Remove(itemVal.Length - 4) : itemVal);
machine.Description = (itemVal.ToLowerInvariant().EndsWith(".zip") ? itemVal.Remove(itemVal.Length - 4) : itemVal);
break;
}
}
// Handle any internal items
else if (cmpr.RowType == CmpRowType.Internal &&
string.Equals(cmpr.InternalName, "file", StringComparison.OrdinalIgnoreCase) &&
cmpr.Internal != null)
{
containsItems = true;
// Create the proper DatItem based on the type
Rom item = DatItem.Create(ItemType.Rom) as Rom;
// Then populate it with information
item.CopyMachineInformation(machine);
item.Source = new Source
{
Index = indexId,
Name = filename,
};
// Loop through all of the attributes
foreach (var kvp in cmpr.Internal)
{
string attrKey = kvp.Key;
string attrVal = kvp.Value;
switch (attrKey)
{
//If the item is empty, we automatically skip it because it's a fluke
case "":
continue;
// Regular attributes
case "name":
item.Name = attrVal;
break;
case "size":
item.Size = Sanitizer.CleanLong(attrVal);
break;
case "crc":
item.CRC = attrVal;
break;
case "date":
item.Date = attrVal;
break;
}
}
// Now process and add the rom
ParseAddHelper(item);
}
}
// If no items were found for this machine, add a Blank placeholder
if (!containsItems)
{
Blank blank = new Blank()
{
Source = new Source
{
Index = indexId,
Name = filename,
},
};
blank.CopyMachineInformation(machine);
// Now process and add the rom
ParseAddHelper(blank);
}
}
public void CreateItemTypeTest(ItemType?itemType, ItemType expected)
{
var actual = DatItem.Create(itemType);
Assert.Equal(expected, actual.ItemType);
}
/// <summary>
/// Attempt to add a file to the output if it matches
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if the file was used to rebuild, false otherwise</returns>
private static bool RebuildGenericHelper(
DatFile datFile,
string file,
string outDir,
bool quickScan,
bool date,
bool inverse,
OutputFormat outputFormat,
TreatAsFile asFiles)
{
// If we somehow have a null filename, return
if (file == null)
{
return(false);
}
// Set the deletion variables
bool usedExternally = false, usedInternally = false;
// Create an empty list of BaseFile for archive entries
List <BaseFile> entries = null;
// Get the TGZ and TXZ status for later
GZipArchive tgz = new GZipArchive(file);
XZArchive txz = new XZArchive(file);
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))
{
BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles);
// Create the correct DatItem
DatItem internalDatItem;
if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
internalDatItem = new Media(internalFileInfo);
}
else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
internalDatItem = new Disk(internalFileInfo);
}
else
{
internalDatItem = new Rom(internalFileInfo);
}
usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat);
}
// Otherwise, loop through the entries and try to match
else
{
foreach (BaseFile entry in entries)
{
DatItem internalDatItem = DatItem.Create(entry);
usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */);
}
}
return(usedExternally || usedInternally);
}
