/// <summary> /// Detect and replace header(s) to the given file /// </summary> /// <param name="file">Name of the file to be parsed</param> /// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param> /// <returns>True if a header was found and appended, false otherwise</returns> public bool RestoreHeader(string file, string outDir) { // Create the output directory if it doesn't exist if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); } // First, get the SHA-1 hash of the file BaseFile baseFile = BaseFile.GetInfo(file, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive); // Retrieve a list of all related headers from the database List <string> headers = RetrieveHeadersFromDatabase(Utilities.ByteArrayToString(baseFile.SHA1)); // If we have nothing retrieved, we return false if (headers.Count == 0) { return(false); } // Now loop through and create the reheadered files, if possible for (int i = 0; i < headers.Count; i++) { string outputFile = (string.IsNullOrWhiteSpace(outDir) ? $"{Path.GetFullPath(file)}.new" : Path.Combine(outDir, Path.GetFileName(file))) + i; logger.User($"Creating reheadered file: {outputFile}"); AppendBytes(file, outputFile, Utilities.StringToByteArray(headers[i]), null); logger.User("Reheadered file created!"); } return(true); }
/// <summary> /// Process a single file as a file /// </summary> /// <param name="datFile">Current DatFile object to add to</param> /// <param name="item">File to be added</param> /// <param name="basePath">Path the represents the parent directory</param> /// <param name="hashes">Hashes to include in the information</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles) { logger.Verbose($"'{Path.GetFileName(item)}' treated like a file"); BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles); DatItem datItem = DatItem.Create(baseFile); ProcessFileHelper(datFile, item, datItem, basePath, string.Empty); }
/// <summary> /// Detect header skipper compliance and create an output file /// </summary> /// <param name="file">Name of the file to be parsed</param> /// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param> /// <param name="nostore">True if headers should not be stored in the database, false otherwise</param> /// <returns>True if the output file was created, false otherwise</returns> private bool DetectTransformStore(string file, string outDir, bool nostore) { // Create the output directory if it doesn't exist if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); } logger.User($"\nGetting skipper information for '{file}'"); // Get the skipper rule that matches the file, if any SkipperMatch.Init(); SkipperRule rule = SkipperMatch.GetMatchingRule(file, string.Empty); // If we have an empty rule, return false if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None) { return(false); } logger.User("File has a valid copier header"); // Get the header bytes from the file first string hstr; try { // Extract the header as a string for the database using var fs = File.OpenRead(file); byte[] hbin = new byte[(int)rule.StartOffset]; fs.Read(hbin, 0, (int)rule.StartOffset); hstr = Utilities.ByteArrayToString(hbin); } catch { return(false); } // Apply the rule to the file string newfile = (string.IsNullOrWhiteSpace(outDir) ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file))); rule.TransformFile(file, newfile); // If the output file doesn't exist, return false if (!File.Exists(newfile)) { return(false); } // Now add the information to the database if it's not already there if (!nostore) { BaseFile baseFile = BaseFile.GetInfo(newfile, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive); AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile); } return(true); }
public void GetInfoNormalFileTest() { // Get the path to the test data string filename = Path.Combine(Environment.CurrentDirectory, "TestData", "file-to-hash.bin"); // Set all of the expected hash values string expectedCrc = "ba02a660"; string expectedMd5 = "b722871eaa950016296184d026c5dec9"; string expectedSha1 = "eea1ee2d801d830c4bdad4df3c8da6f9f52d1a9f"; string expectedSha256 = "fdb02dee8c319c52087382c45f099c90d0b6cc824850aff28c1bfb2884b7b855"; string expectedSha384 = "e276c49618fff25bc1fe2e0659cd0ef0e7c1186563b063e07c52323b9899f3ce9b091be04d6208444b3ef1265e879074"; string expectedSha512 = "15d69514eb628c2403e945a7cafd1d27e557f6e336c69b63ea17e7ed9d256cc374ee662f09305836d6de37fdae59d83883b982aa8446e4ff26346b6b6b50b240"; string expectedSpamSum = "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERk:hZRdxZENFs+rPSromek"; // Get the BaseFile generated from the path var baseFile = BaseFile.GetInfo(filename, hashes: Hash.All); // Extract all the hashes to string string actualCrc = Utilities.ByteArrayToString(baseFile.CRC); string actualMd5 = Utilities.ByteArrayToString(baseFile.MD5); string actualSha1 = Utilities.ByteArrayToString(baseFile.SHA1); string actualSha256 = Utilities.ByteArrayToString(baseFile.SHA256); string actualSha384 = Utilities.ByteArrayToString(baseFile.SHA384); string actualSha512 = Utilities.ByteArrayToString(baseFile.SHA512); string actualSpamSum = Encoding.UTF8.GetString(baseFile.SpamSum); // Verify all of the hashes match Assert.Equal(expectedCrc, actualCrc); Assert.Equal(expectedMd5, actualMd5); Assert.Equal(expectedSha1, actualSha1); Assert.Equal(expectedSha256, actualSha256); Assert.Equal(expectedSha384, actualSha384); Assert.Equal(expectedSha512, actualSha512); Assert.Equal(expectedSpamSum, actualSpamSum); // Convert to Rom for sanity check var rom = new Rom(baseFile); // Verify all of the hashes match Assert.Equal(expectedCrc, rom.CRC); Assert.Equal(expectedMd5, rom.MD5); Assert.Equal(expectedSha1, rom.SHA1); Assert.Equal(expectedSha256, rom.SHA256); Assert.Equal(expectedSha384, rom.SHA384); Assert.Equal(expectedSha512, rom.SHA512); Assert.Equal(expectedSpamSum, rom.SpamSum); }
/// <summary> /// Get the rebuild state for a given item /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="datItem">Information for the current file to rebuild from</param> /// <param name="stream">Stream representing the input file</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="dupes">Output list of duplicate items to rebuild to</param> /// <returns>True if the item should be rebuilt, false otherwise</returns> private static bool ShouldRebuild(DatFile datFile, DatItem datItem, Stream stream, bool inverse, out ConcurrentList <DatItem> dupes) { // Find if the file has duplicates in the DAT dupes = datFile.Items.GetDuplicates(datItem); bool hasDuplicates = dupes.Count > 0; // If we have duplicates but we're filtering if (hasDuplicates && inverse) { return(false); } // If we have duplicates without filtering else if (hasDuplicates && !inverse) { return(true); } // If we have no duplicates and we're filtering else if (!hasDuplicates && inverse) { string machinename = null; // Get the item from the current file Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true)); item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name); item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name); // If we are coming from an archive, set the correct machine name if (machinename != null) { item.Machine.Name = machinename; item.Machine.Description = machinename; } dupes.Add(item); return(true); } // If we have no duplicates and we're not filtering else { return(false); } }
/// <summary> /// Gets all valid DATs that match in the DAT root /// </summary> /// <param name="inputs">List of input strings to check for, presumably file names</param> /// <returns>Dictionary of hash/full path for each of the valid DATs</returns> internal Dictionary <string, string> GetValidDats(List <string> inputs) { // Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not List <string> datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList(); List <string> lowerCaseDats = datRootDats.ConvertAll(i => Path.GetFileName(i).ToLowerInvariant()); Dictionary <string, string> foundDats = new Dictionary <string, string>(); foreach (string input in inputs) { if (lowerCaseDats.Contains(input.ToLowerInvariant())) { string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]); string sha1 = Utilities.ByteArrayToString(BaseFile.GetInfo(fullpath, hashes: Hash.SHA1).SHA1); foundDats.Add(sha1, fullpath); } else { logger.Warning($"The file '{input}' could not be found in the DAT root"); } } return(foundDats); }
/// <summary> /// Find duplicates and rebuild individual files to output /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="datItem">Information for the current file to rebuild from</param> /// <param name="file">Name of the file to process</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param> /// <returns>True if the file was able to be rebuilt, false otherwise</returns> private static bool RebuildIndividualFile( DatFile datFile, DatItem datItem, string file, string outDir, bool date, bool inverse, OutputFormat outputFormat, bool?isZip = null) { // Set the initial output value bool rebuilt = false; // If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) && !(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) && !(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba)) { outputFormat = OutputFormat.Folder; } // If we have a Disk or Media, change it into a Rom for later use if (datItem.ItemType == ItemType.Disk) { datItem = (datItem as Disk).ConvertToRom(); } else if (datItem.ItemType == ItemType.Media) { datItem = (datItem as Media).ConvertToRom(); } // Prepopluate a key string string crc = (datItem as Rom).CRC ?? string.Empty; // Try to get the stream for the file if (!GetFileStream(datItem, file, isZip, out Stream fileStream)) { return(false); } // If either we have duplicates or we're filtering if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes)) { // If we have a very specific TGZ->TGZ case, just copy it accordingly if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip)) { return(true); } // If we have a very specific TXZ->TXZ case, just copy it accordingly if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip)) { return(true); } logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly..."); rebuilt = true; // Special case for partial packing mode bool shouldCheck = false; if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial) { shouldCheck = true; datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false); } // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { // If we should check for the items in the machine if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1) { outputFormat = OutputFormat.Folder; } else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1) { outputFormat = OutputFormat.ParentFolder; } // Get the output archive, if possible Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat); // Now rebuild to the output file outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile()); } // Close the input stream fileStream?.Dispose(); } // Now we want to take care of headers, if applicable if (datFile.Header.HeaderSkipper != null) { // Check to see if we have a matching header first SkipperMatch.Init(); SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper)); // If there's a match, create the new file to write if (rule.Tests != null && rule.Tests.Count != 0) { // If the file could be transformed correctly MemoryStream transformStream = new MemoryStream(); if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true)) { // Get the file informations that we will be using Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true)); // If we have duplicates and we're not filtering if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes)) { logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly..."); rebuilt = true; // Now loop through the list and rebuild accordingly foreach (DatItem item in dupes) { // Create a headered item to use as well datItem.CopyMachineInformation(item); datItem.SetName($"{datItem.GetName()}_{crc}"); // Get the output archive, if possible Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat); // Now rebuild to the output file bool eitherSuccess = false; eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile()); eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile()); // Now add the success of either rebuild rebuilt &= eitherSuccess; } } } // Dispose of the stream transformStream?.Dispose(); } // Dispose of the stream fileStream?.Dispose(); } return(rebuilt); }
/// <summary> /// Attempt to add a file to the output if it matches /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="file">Name of the file to process</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>True if the file was used to rebuild, false otherwise</returns> private static bool RebuildGenericHelper( DatFile datFile, string file, string outDir, bool quickScan, bool date, bool inverse, OutputFormat outputFormat, TreatAsFile asFiles) { // If we somehow have a null filename, return if (file == null) { return(false); } // Set the deletion variables bool usedExternally = false, usedInternally = false; // Create an empty list of BaseFile for archive entries List <BaseFile> entries = null; // Get the TGZ and TXZ status for later GZipArchive tgz = new GZipArchive(file); XZArchive txz = new XZArchive(file); bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent(); // Get the base archive first BaseArchive archive = BaseArchive.Create(file); // Now get all extracted items from the archive if (archive != null) { archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard; entries = archive.GetChildren(); } // If the entries list is null, we encountered an error or have a file and should scan externally if (entries == null && File.Exists(file)) { BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles); // Create the correct DatItem DatItem internalDatItem; if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat)) { internalDatItem = new Media(internalFileInfo); } else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) { internalDatItem = new Disk(internalFileInfo); } else { internalDatItem = new Rom(internalFileInfo); } usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat); } // Otherwise, loop through the entries and try to match else { foreach (BaseFile entry in entries) { DatItem internalDatItem = DatItem.Create(entry); usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */); } } return(usedExternally || usedInternally); }