/// <summary> /// Create a new Dat from a directory /// </summary> /// <param name="datFile">Current DatFile object to add to</param> /// <param name="basePath">Base folder to be used in creating the DAT</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> /// <param name="skipFileType">Type of files that should be skipped</param> /// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param> /// <param name="hashes">Hashes to include in the information</param> public static bool PopulateFromDir( DatFile datFile, string basePath, TreatAsFile asFiles = 0x00, SkipFileType skipFileType = SkipFileType.None, bool addBlanks = false, Hash hashes = Hash.Standard) { // Set the progress variables long totalSize = 0; long currentSize = 0; InternalStopwatch watch = new InternalStopwatch($"Populating DAT from {basePath}"); // Process the input if (Directory.Exists(basePath)) { logger.Verbose($"Folder found: {basePath}"); // Get a list of all files to process List <string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList(); // Loop through and add the file sizes Parallel.ForEach(files, Globals.ParallelOptions, item => { Interlocked.Add(ref totalSize, new FileInfo(item).Length); }); // Process the files in the main folder or any subfolder logger.User(totalSize, currentSize); foreach (string item in files) { CheckFileForHashes(datFile, item, basePath, asFiles, skipFileType, addBlanks, hashes); currentSize += new FileInfo(item).Length; logger.User(totalSize, currentSize, item); } // Now find all folders that are empty, if we are supposed to if (addBlanks) { ProcessDirectoryBlanks(datFile, basePath); } } else if (File.Exists(basePath)) { logger.Verbose($"File found: {basePath}"); totalSize = new FileInfo(basePath).Length; logger.User(totalSize, currentSize); string parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath)); CheckFileForHashes(datFile, basePath, parentPath, asFiles, skipFileType, addBlanks, hashes); logger.User(totalSize, totalSize, basePath); } watch.Stop(); return(true); }
/// <summary> /// Retrieve file information for a single file /// </summary> /// <param name="input">Filename to get information from</param> /// <param name="header">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param> /// <param name="hashes">Hashes to include in the information</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>Populated BaseFile object if success, empty one on error</returns> public static BaseFile GetInfo(string input, string header = null, Hash hashes = Hash.Standard, TreatAsFile asFiles = 0x00) { // Add safeguard if file doesn't exist if (!File.Exists(input)) { return(null); } // Get input information var fileType = GetFileType(input); Stream inputStream = File.OpenRead(input); // Try to match the supplied header skipper if (header != null) { SkipperMatch.Init(); var rule = SkipperMatch.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header)); // If there's a match, transform the stream before getting info if (rule.Tests != null && rule.Tests.Count != 0) { // Create the output stream MemoryStream outputStream = new MemoryStream(); // Transform the stream and get the information from it rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true); inputStream = outputStream; } } // Get the info in the proper manner BaseFile baseFile; if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat)) { baseFile = AaruFormat.Create(inputStream); } else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) { baseFile = CHDFile.Create(inputStream); } else { baseFile = GetInfo(inputStream, hashes: hashes, keepReadOpen: false); } // Dispose of the input stream inputStream?.Dispose(); // Add unique data from the file baseFile.Filename = Path.GetFileName(input); baseFile.Date = new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss"); return(baseFile); }
/// <summary> /// Process a single file as a file /// </summary> /// <param name="datFile">Current DatFile object to add to</param> /// <param name="item">File to be added</param> /// <param name="basePath">Path the represents the parent directory</param> /// <param name="hashes">Hashes to include in the information</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles) { logger.Verbose($"'{Path.GetFileName(item)}' treated like a file"); BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles); DatItem datItem = DatItem.Create(baseFile); ProcessFileHelper(datFile, item, datItem, basePath, string.Empty); }
public override bool ProcessFeatures(Dictionary <string, Help.Feature> features) { // If the base fails, just fail out if (!base.ProcessFeatures(features)) { return(false); } // Get feature flags bool addBlankFiles = GetBoolean(features, AddBlankFilesValue); bool addFileDates = GetBoolean(features, AddDateValue); TreatAsFile asFiles = GetTreatAsFiles(features); bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue); var includeInScan = GetIncludeInScan(features); var skipFileType = GetSkipFileType(features); // Apply the specialized field removals to the cleaner if (!addFileDates) { Remover.PopulateExclusionsFromList(new List <string> { "DatItem.Date" }); } // Create a new DATFromDir object and process the inputs DatFile basedat = DatFile.Create(Header); basedat.Header.Date = DateTime.Now.ToString("yyyy-MM-dd"); // For each input directory, create a DAT foreach (string path in Inputs) { if (Directory.Exists(path) || File.Exists(path)) { // Clone the base Dat for information DatFile datdata = DatFile.Create(basedat.Header); // Get the base path and fill the header, if needed string basePath = Path.GetFullPath(path); datdata.FillHeaderFromPath(basePath, noAutomaticDate); // Now populate from the path bool success = DatTools.DatFromDir.PopulateFromDir( datdata, basePath, asFiles, skipFileType, addBlankFiles, hashes: includeInScan); if (success) { // Perform additional processing steps Extras.ApplyExtras(datdata); Splitter.ApplySplitting(datdata, false); Filter.ApplyFilters(datdata); Cleaner.ApplyCleaning(datdata); Remover.ApplyRemovals(datdata); // Write out the file Writer.Write(datdata, OutputDir); } else { Console.WriteLine(); OutputRecursive(0); } } } return(true); }
/// <summary> /// Check a given file for hashes, based on current settings /// </summary> /// <param name="datFile">Current DatFile object to add to</param> /// <param name="item">Filename of the item to be checked</param> /// <param name="basePath">Base folder to be used in creating the DAT</param> /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param> /// <param name="skipFileType">Type of files that should be skipped</param> /// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param> /// <param name="hashes">Hashes to include in the information</param> private static void CheckFileForHashes( DatFile datFile, string item, string basePath, TreatAsFile asFiles, SkipFileType skipFileType, bool addBlanks, Hash hashes) { // If we're in depot mode, process it separately if (CheckDepotFile(datFile, item)) { return; } // Initialize possible archive variables BaseArchive archive = BaseArchive.Create(item); // Process archives according to flags if (archive != null) { // Set the archive flags archive.AvailableHashes = hashes; // Skip if we're treating archives as files and skipping files if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File) { return; } // Skip if we're skipping archives else if (skipFileType == SkipFileType.Archive) { return; } // Process as archive if we're not treating archives as files else if (!asFiles.HasFlag(TreatAsFile.Archive)) { var extracted = archive.GetChildren(); // If we have internal items to process, do so if (extracted != null) { ProcessArchive(datFile, item, basePath, extracted); } // Now find all folders that are empty, if we are supposed to if (addBlanks) { ProcessArchiveBlanks(datFile, item, basePath, archive); } } // Process as file if we're treating archives as files else { ProcessFile(datFile, item, basePath, hashes, asFiles); } } // Process non-archives according to flags else { // Skip if we're skipping files if (skipFileType == SkipFileType.File) { return; } // Process as file else { ProcessFile(datFile, item, basePath, hashes, asFiles); } } }
public override void ProcessFeatures(Dictionary <string, Feature> features) { base.ProcessFeatures(features); // Get a list of files from the input datfiles var datfiles = GetList(features, DatListValue); var datfilePaths = DirectoryExtensions.GetFilesOnly(datfiles); // Get feature flags TreatAsFile asFiles = GetTreatAsFiles(features); bool hashOnly = GetBoolean(features, HashOnlyValue); bool quickScan = GetBoolean(features, QuickValue); var splitType = GetSplitType(features); // If we are in individual mode, process each DAT on their own if (GetBoolean(features, IndividualValue)) { foreach (ParentablePath datfile in datfilePaths) { // Parse in from the file DatFile datdata = DatFile.Create(); datdata.Parse(datfile, int.MaxValue, keep: true); // Perform additional processing steps datdata.ApplyExtras(Extras); datdata.ApplySplitting(splitType, true); datdata.ApplyFilter(Filter); datdata.ApplyCleaning(Cleaner); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) { datdata.Header.HeaderSkipper = Header.HeaderSkipper; } // If we have the depot flag, respect it if (Header.InputDepot?.IsActive ?? false) { datdata.VerifyDepot(Inputs); } else { // Loop through and add the inputs to check against logger.User("Processing files:\n"); foreach (string input in Inputs) { datdata.PopulateFromDir(input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard); } datdata.VerifyGeneric(hashOnly); } // Now write out if there are any items left datdata.WriteStatsToConsole(); datdata.Write(OutputDir); } } // Otherwise, process all DATs into the same output else { InternalStopwatch watch = new InternalStopwatch("Populating internal DAT"); // Add all of the input DATs into one huge internal DAT DatFile datdata = DatFile.Create(); foreach (ParentablePath datfile in datfilePaths) { datdata.Parse(datfile, int.MaxValue, keep: true); } // Perform additional processing steps datdata.ApplyExtras(Extras); datdata.ApplySplitting(splitType, true); datdata.ApplyFilter(Filter); datdata.ApplyCleaning(Cleaner); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) { datdata.Header.HeaderSkipper = Header.HeaderSkipper; } watch.Stop(); // If we have the depot flag, respect it if (Header.InputDepot?.IsActive ?? false) { datdata.VerifyDepot(Inputs); } else { // Loop through and add the inputs to check against logger.User("Processing files:\n"); foreach (string input in Inputs) { datdata.PopulateFromDir(input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard); } datdata.VerifyGeneric(hashOnly); } // Now write out if there are any items left datdata.WriteStatsToConsole(); datdata.Write(OutputDir); } }
public override void ProcessFeatures(Dictionary <string, Feature> features) { base.ProcessFeatures(features); // Get feature flags TreatAsFile asFiles = GetTreatAsFiles(features); bool date = GetBoolean(features, AddDateValue); bool delete = GetBoolean(features, DeleteValue); bool inverse = GetBoolean(features, InverseValue); bool quickScan = GetBoolean(features, QuickValue); bool updateDat = GetBoolean(features, UpdateDatValue); var outputFormat = GetOutputFormat(features); // If we have the romba flag if (Header.OutputDepot?.IsActive == true) { // Update TorrentGzip output if (outputFormat == OutputFormat.TorrentGzip) { outputFormat = OutputFormat.TorrentGzipRomba; } // Update TorrentXz output else if (outputFormat == OutputFormat.TorrentXZ) { outputFormat = OutputFormat.TorrentXZRomba; } } // Get a list of files from the input datfiles var datfiles = GetList(features, DatListValue); var datfilePaths = DirectoryExtensions.GetFilesOnly(datfiles); // If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir if (GetBoolean(features, IndividualValue)) { foreach (ParentablePath datfile in datfilePaths) { DatFile datdata = DatFile.Create(); datdata.Parse(datfile, int.MaxValue, keep: true); // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; datdata.Header.OutputDepot = Header.OutputDepot.Clone() as DepotInformation; // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) { datdata.Header.HeaderSkipper = Header.HeaderSkipper; } // If we have the depot flag, respect it bool success; if (Header.InputDepot?.IsActive ?? false) { success = datdata.RebuildDepot(Inputs, Path.Combine(OutputDir, datdata.Header.FileName), date, delete, inverse, outputFormat); } else { success = datdata.RebuildGeneric(Inputs, Path.Combine(OutputDir, datdata.Header.FileName), quickScan, date, delete, inverse, outputFormat, asFiles); } // If we have a success and we're updating the DAT, write it out if (success && updateDat) { datdata.Header.FileName = $"fixDAT_{Header.FileName}"; datdata.Header.Name = $"fixDAT_{Header.Name}"; datdata.Header.Description = $"fixDAT_{Header.Description}"; datdata.Items.ClearMarked(); datdata.Write(OutputDir); } } } // Otherwise, process all DATs into the same output else { InternalStopwatch watch = new InternalStopwatch("Populating internal DAT"); // Add all of the input DATs into one huge internal DAT DatFile datdata = DatFile.Create(); foreach (ParentablePath datfile in datfilePaths) { datdata.Parse(datfile, int.MaxValue, keep: true); } // Set depot information datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation; datdata.Header.OutputDepot = Header.OutputDepot.Clone() as DepotInformation; // If we have overridden the header skipper, set it now if (!string.IsNullOrEmpty(Header.HeaderSkipper)) { datdata.Header.HeaderSkipper = Header.HeaderSkipper; } watch.Stop(); // If we have the depot flag, respect it bool success; if (Header.InputDepot?.IsActive ?? false) { success = datdata.RebuildDepot(Inputs, OutputDir, date, delete, inverse, outputFormat); } else { success = datdata.RebuildGeneric(Inputs, OutputDir, quickScan, date, delete, inverse, outputFormat, asFiles); } // If we have a success and we're updating the DAT, write it out if (success && updateDat) { datdata.Header.FileName = $"fixDAT_{Header.FileName}"; datdata.Header.Name = $"fixDAT_{Header.Name}"; datdata.Header.Description = $"fixDAT_{Header.Description}"; datdata.Items.ClearMarked(); datdata.Write(OutputDir); } } }
public override void ProcessFeatures(Dictionary <string, Library.Help.Feature> features) { base.ProcessFeatures(features); // Get feature flags bool addBlankFiles = GetBoolean(features, AddBlankFilesValue); bool addFileDates = GetBoolean(features, AddDateValue); TreatAsFile asFiles = GetTreatAsFiles(features); bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue); var includeInScan = GetIncludeInScan(features); var skipFileType = GetSkipFileType(features); var splitType = GetSplitType(features); // Apply the specialized field removals to the cleaner if (Cleaner.ExcludeFields == null) { Cleaner.ExcludeFields = new List <Field>(); } if (!addFileDates) { Cleaner.ExcludeFields.Add(Field.DatItem_Date); } // Create a new DATFromDir object and process the inputs DatFile basedat = DatFile.Create(Header); basedat.Header.Date = DateTime.Now.ToString("yyyy-MM-dd"); // For each input directory, create a DAT foreach (string path in Inputs) { if (Directory.Exists(path) || File.Exists(path)) { // Clone the base Dat for information DatFile datdata = DatFile.Create(basedat.Header); // Get the base path and fill the header, if needed string basePath = Path.GetFullPath(path); datdata.FillHeaderFromPath(basePath, noAutomaticDate); // Now populate from the path bool success = datdata.PopulateFromDir( basePath, asFiles, skipFileType, addBlankFiles, hashes: includeInScan); if (success) { // Perform additional processing steps datdata.ApplyExtras(Extras); datdata.ApplySplitting(splitType, false); datdata.ApplyFilter(Filter); datdata.ApplyCleaning(Cleaner); // Write out the file datdata.Write(OutputDir); } else { Console.WriteLine(); OutputRecursive(0); } } } }
/// <summary> /// Attempt to add a file to the output if it matches /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="file">Name of the file to process</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>True if the file was used to rebuild, false otherwise</returns> private static bool RebuildGenericHelper( DatFile datFile, string file, string outDir, bool quickScan, bool date, bool inverse, OutputFormat outputFormat, TreatAsFile asFiles) { // If we somehow have a null filename, return if (file == null) { return(false); } // Set the deletion variables bool usedExternally = false, usedInternally = false; // Create an empty list of BaseFile for archive entries List <BaseFile> entries = null; // Get the TGZ and TXZ status for later GZipArchive tgz = new GZipArchive(file); XZArchive txz = new XZArchive(file); bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent(); // Get the base archive first BaseArchive archive = BaseArchive.Create(file); // Now get all extracted items from the archive if (archive != null) { archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard; entries = archive.GetChildren(); } // If the entries list is null, we encountered an error or have a file and should scan externally if (entries == null && File.Exists(file)) { BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles); // Create the correct DatItem DatItem internalDatItem; if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat)) { internalDatItem = new Media(internalFileInfo); } else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD)) { internalDatItem = new Disk(internalFileInfo); } else { internalDatItem = new Rom(internalFileInfo); } usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat); } // Otherwise, loop through the entries and try to match else { foreach (BaseFile entry in entries) { DatItem internalDatItem = DatItem.Create(entry); usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */); } } return(usedExternally || usedInternally); }
/// <summary> /// Process the DAT and find all matches in input files and folders /// </summary> /// <param name="datFile">Current DatFile object to rebuild from</param> /// <param name="inputs">List of input files/folders to check</param> /// <param name="outDir">Output directory to use to build to</param> /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param> /// <param name="date">True if the date from the DAT should be used if available, false otherwise</param> /// <param name="delete">True if input files should be deleted, false otherwise</param> /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param> /// <param name="outputFormat">Output format that files should be written to</param> /// <param name="asFiles">TreatAsFiles representing special format scanning</param> /// <returns>True if rebuilding was a success, false otherwise</returns> public static bool RebuildGeneric( DatFile datFile, List <string> inputs, string outDir, bool quickScan = false, bool date = false, bool delete = false, bool inverse = false, OutputFormat outputFormat = OutputFormat.Folder, TreatAsFile asFiles = 0x00) { #region Perform setup // If the DAT is not populated and inverse is not set, inform the user and quit if (datFile.Items.TotalCount == 0 && !inverse) { logger.User("No entries were found to rebuild, exiting..."); return(false); } // Check that the output directory exists if (!Directory.Exists(outDir)) { Directory.CreateDirectory(outDir); outDir = Path.GetFullPath(outDir); } // Now we want to get forcepack flag if it's not overridden if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None) { outputFormat = GetOutputFormat(datFile.Header.ForcePacking); } #endregion bool success = true; #region Rebuild from sources in order string format = FromOutputFormat(outputFormat) ?? string.Empty; InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}"); // Now loop through all of the files in all of the inputs foreach (string input in inputs) { // If the input is a file if (File.Exists(input)) { logger.User($"Checking file: {input}"); bool rebuilt = RebuildGenericHelper(datFile, input, outDir, quickScan, date, inverse, outputFormat, asFiles); // If we are supposed to delete the file, do so if (delete && rebuilt) { File.Delete(input); } } // If the input is a directory else if (Directory.Exists(input)) { logger.Verbose($"Checking directory: {input}"); foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) { logger.User($"Checking file: {file}"); bool rebuilt = RebuildGenericHelper(datFile, file, outDir, quickScan, date, inverse, outputFormat, asFiles); // If we are supposed to delete the file, do so if (delete && rebuilt) { File.Delete(input); } } } } watch.Stop(); #endregion return(success); }