public void CreateBaseFileTest(FileType fileType, ItemType?expected) { var baseFile = CreateBaseFile(fileType); var actual = DatItem.Create(baseFile); Assert.Equal(expected, actual?.ItemType); }
/// <inheritdoc/> public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false) { // Open a file reader Encoding enc = filename.GetEncoding(); SeparatedValueReader svr = new SeparatedValueReader(File.OpenRead(filename), enc) { Header = true, Quotes = true, Separator = _delim, VerifyFieldCount = true, }; // If we're somehow at the end of the stream already, we can't do anything if (svr.EndOfStream) { return; } // Read in the header svr.ReadHeader(); // Loop through all of the data lines while (!svr.EndOfStream) { try { // Get the current line, split and parse svr.ReadNextLine(); // Create mapping dictionaries Setter setter = new Setter(); setter.PopulateSettersFromList(svr.HeaderValues, svr.Line); // Set DatHeader fields DatHeader datHeader = new DatHeader(); setter.SetFields(datHeader); Header.ConditionalCopy(datHeader); // Set Machine and DatItem fields if (setter.DatItemMappings.ContainsKey(DatItemField.Type)) { DatItem datItem = DatItem.Create(setter.DatItemMappings[DatItemField.Type].AsItemType()); setter.SetFields(datItem); datItem.Machine = new Machine(); setter.SetFields(datItem.Machine); datItem.Source = new Source(indexId, filename); ParseAddHelper(datItem, statsOnly); } } catch (Exception ex) when(!throwOnError) { string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'"; logger.Error(ex, message); } } svr.Dispose(); }
/// <summary> /// Process a single file as a file /// </summary> /// <param name="datFile">Current DatFile object to add to</param> /// <param name="item">File to be added</param> /// <param name="basePath">Path the represents the parent directory</param> /// <param name="hashes">Hashes to include in the information</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles) { logger.Verbose($"'{Path.GetFileName(item)}' treated like a file"); BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles); DatItem datItem = DatItem.Create(baseFile); ProcessFileHelper(datFile, item, datItem, basePath, string.Empty); }
/// <summary> /// Process a single file as an archive /// </summary> /// <param name="datFile">Current DatFile object to add to</param> /// <param name="item">File to be added</param> /// <param name="basePath">Path the represents the parent directory</param> /// <param name="extracted">List of BaseFiles representing the internal files</param> private static void ProcessArchive(DatFile datFile, string item, string basePath, List <BaseFile> extracted) { // Get the parent path for all items string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item); // First take care of the found items Parallel.ForEach(extracted, Globals.ParallelOptions, baseFile => { DatItem datItem = DatItem.Create(baseFile); ProcessFileHelper(datFile, item, datItem, basePath, parent); }); }
/// <summary> /// Parse a character-separated value DAT and return all found games and roms within /// </summary> /// <param name="filename">Name of the file to be parsed</param> /// <param name="indexId">Index ID for the DAT</param> /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param> /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param> protected override void ParseFile(string filename, int indexId, bool keep, bool throwOnError = false) { // Open a file reader Encoding enc = FileExtensions.GetEncoding(filename); SeparatedValueReader svr = new SeparatedValueReader(FileExtensions.TryOpenRead(filename), enc) { Header = true, Quotes = true, Separator = _delim, VerifyFieldCount = true, }; // If we're somehow at the end of the stream already, we can't do anything if (svr.EndOfStream) { return; } // Read in the header svr.ReadHeader(); // Loop through all of the data lines while (!svr.EndOfStream) { try { // Get the current line, split and parse svr.ReadNextLine(); // Create mapping dictionary var mappings = new Dictionary <Field, string>(); // Now we loop through and get values for everything for (int i = 0; i < svr.HeaderValues.Count; i++) { Field key = svr.HeaderValues[i].AsField(); string value = svr.Line[i]; mappings[key] = value; } // Set DatHeader fields DatHeader header = new DatHeader(); header.SetFields(mappings); Header.ConditionalCopy(header); // Set Machine and DatItem fields if (mappings.ContainsKey(Field.DatItem_Type)) { DatItem datItem = DatItem.Create(mappings[Field.DatItem_Type].AsItemType()); datItem.SetFields(mappings); datItem.Source = new Source(indexId, filename); ParseAddHelper(datItem); } } catch (Exception ex) { string message = $"'{filename}' - There was an error parsing line {svr.LineNumber} '{svr.CurrentLine}'"; logger.Error(ex, message); if (throwOnError) { svr.Dispose(); throw new Exception(message, ex); } } } svr.Dispose(); }
/// <summary> /// Read set information /// </summary> /// <param name="cmpr">ClrMameProReader to use to parse the header</param> /// <param name="resource">True if the item is a resource (bios), false otherwise</param> /// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param> /// <param name="filename">Name of the file to be parsed</param> /// <param name="indexId">Index ID for the DAT</param> private void ReadSet( ClrMameProReader cmpr, bool resource, bool statsOnly, // Standard Dat parsing string filename, int indexId) { // Prepare all internal variables bool containsItems = false; Machine machine = new Machine() { MachineType = (resource ? MachineType.Bios : MachineType.NULL), }; // If there's no subtree to the header, skip it if (cmpr == null || cmpr.EndOfStream) { return; } // While we don't hit an end element or end of stream while (!cmpr.EndOfStream) { cmpr.ReadNextLine(); // Ignore comments and nothingness if (cmpr.RowType == CmpRowType.None || cmpr.RowType == CmpRowType.Comment) { continue; } // If we reached the end of a section, break if (cmpr.RowType == CmpRowType.EndTopLevel) { break; } // Handle any standalone items if (cmpr.RowType == CmpRowType.Standalone && cmpr.Standalone != null) { string itemKey = cmpr.Standalone?.Key.ToLowerInvariant(); string itemVal = cmpr.Standalone?.Value; switch (itemKey) { case "name": machine.Name = itemVal; break; case "description": machine.Description = itemVal; break; case "year": machine.Year = itemVal; break; case "manufacturer": machine.Manufacturer = itemVal; break; case "category": machine.Category = itemVal; break; case "cloneof": machine.CloneOf = itemVal; break; case "romof": machine.RomOf = itemVal; break; case "sampleof": machine.SampleOf = itemVal; break; } } // Handle any internal items else if (cmpr.RowType == CmpRowType.Internal && !string.IsNullOrWhiteSpace(cmpr.InternalName) && cmpr.Internal != null) { containsItems = true; string itemKey = cmpr.InternalName; // Create the proper DatItem based on the type ItemType itemType = itemKey.AsItemType() ?? ItemType.Rom; DatItem item = DatItem.Create(itemType); // Then populate it with information item.CopyMachineInformation(machine); item.Source.Index = indexId; item.Source.Name = filename; // Loop through all of the attributes foreach (var kvp in cmpr.Internal) { string attrKey = kvp.Key; string attrVal = kvp.Value; switch (attrKey) { //If the item is empty, we automatically skip it because it's a fluke case "": continue; // Regular attributes case "name": item.SetName(attrVal); break; case "size": if (item.ItemType == ItemType.Rom) { (item as Rom).Size = Utilities.CleanLong(attrVal); } break; case "crc": if (item.ItemType == ItemType.Rom) { (item as Rom).CRC = attrVal; } break; case "md5": if (item.ItemType == ItemType.Disk) { (item as Disk).MD5 = attrVal; } else if (item.ItemType == ItemType.Media) { (item as Media).MD5 = attrVal; } else if (item.ItemType == ItemType.Rom) { (item as Rom).MD5 = attrVal; } break; case "sha1": if (item.ItemType == ItemType.Disk) { (item as Disk).SHA1 = attrVal; } else if (item.ItemType == ItemType.Media) { (item as Media).SHA1 = attrVal; } else if (item.ItemType == ItemType.Rom) { (item as Rom).SHA1 = attrVal; } break; case "sha256": if (item.ItemType == ItemType.Media) { (item as Media).SHA256 = attrVal; } else if (item.ItemType == ItemType.Rom) { (item as Rom).SHA256 = attrVal; } break; case "sha384": if (item.ItemType == ItemType.Rom) { (item as Rom).SHA384 = attrVal; } break; case "sha512": if (item.ItemType == ItemType.Rom) { (item as Rom).SHA512 = attrVal; } break; case "spamsum": if (item.ItemType == ItemType.Media) { (item as Media).SpamSum = attrVal; } else if (item.ItemType == ItemType.Rom) { (item as Rom).SpamSum = attrVal; } break; case "status": ItemStatus tempFlagStatus = attrVal.AsItemStatus(); if (item.ItemType == ItemType.Disk) { (item as Disk).ItemStatus = tempFlagStatus; } else if (item.ItemType == ItemType.Rom) { (item as Rom).ItemStatus = tempFlagStatus; } break; case "date": if (item.ItemType == ItemType.Release) { (item as Release).Date = attrVal; } else if (item.ItemType == ItemType.Rom) { (item as Rom).Date = attrVal; } break; case "default": if (item.ItemType == ItemType.BiosSet) { (item as BiosSet).Default = attrVal.AsYesNo(); } else if (item.ItemType == ItemType.Release) { (item as Release).Default = attrVal.AsYesNo(); } break; case "description": if (item.ItemType == ItemType.BiosSet) { (item as BiosSet).Description = attrVal; } break; case "region": if (item.ItemType == ItemType.Release) { (item as Release).Region = attrVal; } break; case "language": if (item.ItemType == ItemType.Release) { (item as Release).Language = attrVal; } break; } } // Now process and add the rom ParseAddHelper(item, statsOnly); } } // If no items were found for this machine, add a Blank placeholder if (!containsItems) { Blank blank = new Blank() { Source = new Source { Index = indexId, Name = filename, }, }; blank.CopyMachineInformation(machine); // Now process and add the rom ParseAddHelper(blank, statsOnly); } }
/// <summary> /// Read set information /// </summary> /// <param name="cmpr">ClrMameProReader to use to parse the header</param> /// <param name="filename">Name of the file to be parsed</param> /// <param name="indexId">Index ID for the DAT</param> private void ReadGame(ClrMameProReader cmpr, string filename, int indexId) { // Prepare all internal variables bool containsItems = false; Machine machine = new Machine() { MachineType = MachineType.NULL, }; // If there's no subtree to the header, skip it if (cmpr == null || cmpr.EndOfStream) { return; } // While we don't hit an end element or end of stream while (!cmpr.EndOfStream) { cmpr.ReadNextLine(); // Ignore comments and nothingness if (cmpr.RowType == CmpRowType.None || cmpr.RowType == CmpRowType.Comment) { continue; } // If we reached the end of a section, break if (cmpr.RowType == CmpRowType.EndTopLevel) { break; } // Handle any standalone items if (cmpr.RowType == CmpRowType.Standalone && cmpr.Standalone != null) { string itemKey = cmpr.Standalone?.Key.ToLowerInvariant(); string itemVal = cmpr.Standalone?.Value; switch (itemKey) { case "name": machine.Name = (itemVal.ToLowerInvariant().EndsWith(".zip") ? itemVal.Remove(itemVal.Length - 4) : itemVal); machine.Description = (itemVal.ToLowerInvariant().EndsWith(".zip") ? itemVal.Remove(itemVal.Length - 4) : itemVal); break; } } // Handle any internal items else if (cmpr.RowType == CmpRowType.Internal && string.Equals(cmpr.InternalName, "file", StringComparison.OrdinalIgnoreCase) && cmpr.Internal != null) { containsItems = true; // Create the proper DatItem based on the type Rom item = DatItem.Create(ItemType.Rom) as Rom; // Then populate it with information item.CopyMachineInformation(machine); item.Source = new Source { Index = indexId, Name = filename, }; // Loop through all of the attributes foreach (var kvp in cmpr.Internal) { string attrKey = kvp.Key; string attrVal = kvp.Value; switch (attrKey) { //If the item is empty, we automatically skip it because it's a fluke case "": continue; // Regular attributes case "name": item.Name = attrVal; break; case "size": item.Size = Sanitizer.CleanLong(attrVal); break; case "crc": item.CRC = attrVal; break; case "date": item.Date = attrVal; break; } } // Now process and add the rom ParseAddHelper(item); } } // If no items were found for this machine, add a Blank placeholder if (!containsItems) { Blank blank = new Blank() { Source = new Source { Index = indexId, Name = filename, }, }; blank.CopyMachineInformation(machine); // Now process and add the rom ParseAddHelper(blank); } }
public void CreateItemTypeTest(ItemType?itemType, ItemType expected) { var actual = DatItem.Create(itemType); Assert.Equal(expected, actual.ItemType); }
/// <summary> /// Attempt to add a file to the output if it matches /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="file">Name of the file to process</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>True if the file was used to rebuild, false otherwise</returns> private static bool RebuildGenericHelper( DatFile datFile, string file, string outDir, bool quickScan, bool date, bool inverse, OutputFormat outputFormat, TreatAsFile asFiles) { // If we somehow have a null filename, return if (file == null) { return(false); } // Set the deletion variables bool usedExternally = false, usedInternally = false; // Create an empty list of BaseFile for archive entries List <BaseFile> entries = null; // Get the TGZ and TXZ status for later GZipArchive tgz = new GZipArchive(file); XZArchive txz = new XZArchive(file); bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent(); // Get the base archive first BaseArchive archive = BaseArchive.Create(file); // Now get all extracted items from the archive if (archive != null) { archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard; entries = archive.GetChildren(); } // If the entries list is null, we encountered an error or have a file and should scan externally if (entries == null && File.Exists(file)) { BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles); // Create the correct DatItem DatItem internalDatItem; if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat)) { internalDatItem = new Media(internalFileInfo); } else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) { internalDatItem = new Disk(internalFileInfo); } else { internalDatItem = new Rom(internalFileInfo); } usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat); } // Otherwise, loop through the entries and try to match else { foreach (BaseFile entry in entries) { DatItem internalDatItem = DatItem.Create(entry); usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */); } } return(usedExternally || usedInternally); }