Ejemplo n.º 1
0
        /// <summary>
        /// Detect header skipper compliance and create an output file
        /// </summary>
        /// <param name="file">Name of the file to be parsed</param>
        /// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param>
        /// <param name="nostore">True if headers should not be stored in the database, false otherwise</param>
        /// <returns>True if the output file was created, false otherwise</returns>
        private bool DetectTransformStore(string file, string outDir, bool nostore)
        {
            // Create the output directory if it doesn't exist
            if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir))
            {
                Directory.CreateDirectory(outDir);
            }

            logger.User($"\nGetting skipper information for '{file}'");

            // Get the skipper rule that matches the file, if any
            SkipperMatch.Init();
            SkipperRule rule = SkipperMatch.GetMatchingRule(file, string.Empty);

            // If we have an empty rule, return false
            if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None)
            {
                return(false);
            }

            logger.User("File has a valid copier header");

            // Get the header bytes from the file first
            string hstr;

            try
            {
                // Extract the header as a string for the database
                using var fs = File.OpenRead(file);
                byte[] hbin = new byte[(int)rule.StartOffset];
                fs.Read(hbin, 0, (int)rule.StartOffset);
                hstr = Utilities.ByteArrayToString(hbin);
            }
            catch
            {
                return(false);
            }

            // Apply the rule to the file
            string newfile = (string.IsNullOrWhiteSpace(outDir) ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file)));

            rule.TransformFile(file, newfile);

            // If the output file doesn't exist, return false
            if (!File.Exists(newfile))
            {
                return(false);
            }

            // Now add the information to the database if it's not already there
            if (!nostore)
            {
                BaseFile baseFile = BaseFile.GetInfo(newfile, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
                AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile);
            }

            return(true);
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Retrieve file information for a single file
        /// </summary>
        /// <param name="input">Filename to get information from</param>
        /// <param name="header">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
        /// <param name="hashes">Hashes to include in the information</param>
        /// <param name="asFiles">TreatAsFiles representing special format scanning</param>
        /// <returns>Populated BaseFile object if success, empty one on error</returns>
        public static BaseFile GetInfo(string input, string header = null, Hash hashes = Hash.Standard, TreatAsFile asFiles = 0x00)
        {
            // Add safeguard if file doesn't exist
            if (!File.Exists(input))
            {
                return(null);
            }

            // Get input information
            var    fileType    = GetFileType(input);
            Stream inputStream = File.OpenRead(input);

            // Try to match the supplied header skipper
            if (header != null)
            {
                SkipperMatch.Init();
                var rule = SkipperMatch.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header));

                // If there's a match, transform the stream before getting info
                if (rule.Tests != null && rule.Tests.Count != 0)
                {
                    // Create the output stream
                    MemoryStream outputStream = new MemoryStream();

                    // Transform the stream and get the information from it
                    rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true);
                    inputStream = outputStream;
                }
            }

            // Get the info in the proper manner
            BaseFile baseFile;

            if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
            {
                baseFile = AaruFormat.Create(inputStream);
            }
            else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
            {
                baseFile = CHDFile.Create(inputStream);
            }
            else
            {
                baseFile = GetInfo(inputStream, hashes: hashes, keepReadOpen: false);
            }

            // Dispose of the input stream
            inputStream?.Dispose();

            // Add unique data from the file
            baseFile.Filename = Path.GetFileName(input);
            baseFile.Date     = new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss");

            return(baseFile);
        }
 public SkipperMatchHeaderTests()
 {
     SkipperMatch.Init(false);
 }
Ejemplo n.º 4
0
        /// <summary>
        /// Find duplicates and rebuild individual files to output
        /// </summary>
        /// <param name="datFile">Current DatFile object to rebuild from</param>
        /// <param name="datItem">Information for the current file to rebuild from</param>
        /// <param name="file">Name of the file to process</param>
        /// <param name="outDir">Output directory to use to build to</param>
        /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
        /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
        /// <param name="outputFormat">Output format that files should be written to</param>
        /// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
        /// <returns>True if the file was able to be rebuilt, false otherwise</returns>
        private static bool RebuildIndividualFile(
            DatFile datFile,
            DatItem datItem,
            string file,
            string outDir,
            bool date,
            bool inverse,
            OutputFormat outputFormat,
            bool?isZip = null)
        {
            // Set the initial output value
            bool rebuilt = false;

            // If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
            if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) &&
                !(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) &&
                !(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
            {
                outputFormat = OutputFormat.Folder;
            }

            // If we have a Disk or Media, change it into a Rom for later use
            if (datItem.ItemType == ItemType.Disk)
            {
                datItem = (datItem as Disk).ConvertToRom();
            }
            else if (datItem.ItemType == ItemType.Media)
            {
                datItem = (datItem as Media).ConvertToRom();
            }

            // Prepopluate a key string
            string crc = (datItem as Rom).CRC ?? string.Empty;

            // Try to get the stream for the file
            if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
            {
                return(false);
            }

            // If either we have duplicates or we're filtering
            if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes))
            {
                // If we have a very specific TGZ->TGZ case, just copy it accordingly
                if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
                {
                    return(true);
                }

                // If we have a very specific TXZ->TXZ case, just copy it accordingly
                if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
                {
                    return(true);
                }

                logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
                rebuilt = true;

                // Special case for partial packing mode
                bool shouldCheck = false;
                if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
                {
                    shouldCheck = true;
                    datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false);
                }

                // Now loop through the list and rebuild accordingly
                foreach (DatItem item in dupes)
                {
                    // If we should check for the items in the machine
                    if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
                    {
                        outputFormat = OutputFormat.Folder;
                    }
                    else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
                    {
                        outputFormat = OutputFormat.ParentFolder;
                    }

                    // Get the output archive, if possible
                    Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);

                    // Now rebuild to the output file
                    outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
                }

                // Close the input stream
                fileStream?.Dispose();
            }

            // Now we want to take care of headers, if applicable
            if (datFile.Header.HeaderSkipper != null)
            {
                // Check to see if we have a matching header first
                SkipperMatch.Init();
                SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));

                // If there's a match, create the new file to write
                if (rule.Tests != null && rule.Tests.Count != 0)
                {
                    // If the file could be transformed correctly
                    MemoryStream transformStream = new MemoryStream();
                    if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
                    {
                        // Get the file informations that we will be using
                        Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));

                        // If we have duplicates and we're not filtering
                        if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
                        {
                            logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
                            rebuilt = true;

                            // Now loop through the list and rebuild accordingly
                            foreach (DatItem item in dupes)
                            {
                                // Create a headered item to use as well
                                datItem.CopyMachineInformation(item);
                                datItem.SetName($"{datItem.GetName()}_{crc}");

                                // Get the output archive, if possible
                                Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);

                                // Now rebuild to the output file
                                bool eitherSuccess = false;
                                eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
                                eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());

                                // Now add the success of either rebuild
                                rebuilt &= eitherSuccess;
                            }
                        }
                    }

                    // Dispose of the stream
                    transformStream?.Dispose();
                }

                // Dispose of the stream
                fileStream?.Dispose();
            }

            return(rebuilt);
        }
 public SkipperRuleTransformTests()
 {
     SkipperMatch.Init(false);
 }
Ejemplo n.º 6
0
 public SkipperMatchInternalTests()
 {
     SkipperMatch.Init(true);
 }