/// <summary>
/// Create a new Dat from a directory
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
public static bool PopulateFromDir(
DatFile datFile,
string basePath,
TreatAsFile asFiles = 0x00,
SkipFileType skipFileType = SkipFileType.None,
bool addBlanks = false,
Hash hashes = Hash.Standard)
{
// Set the progress variables
long totalSize = 0;
long currentSize = 0;
InternalStopwatch watch = new InternalStopwatch($"Populating DAT from {basePath}");
// Process the input
if (Directory.Exists(basePath))
{
logger.Verbose($"Folder found: {basePath}");
// Get a list of all files to process
List <string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList();
// Loop through and add the file sizes
Parallel.ForEach(files, Globals.ParallelOptions, item =>
{
Interlocked.Add(ref totalSize, new FileInfo(item).Length);
});
// Process the files in the main folder or any subfolder
logger.User(totalSize, currentSize);
foreach (string item in files)
{
CheckFileForHashes(datFile, item, basePath, asFiles, skipFileType, addBlanks, hashes);
currentSize += new FileInfo(item).Length;
logger.User(totalSize, currentSize, item);
}
// Now find all folders that are empty, if we are supposed to
if (addBlanks)
{
ProcessDirectoryBlanks(datFile, basePath);
}
}
else if (File.Exists(basePath))
{
logger.Verbose($"File found: {basePath}");
totalSize = new FileInfo(basePath).Length;
logger.User(totalSize, currentSize);
string parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath));
CheckFileForHashes(datFile, basePath, parentPath, asFiles, skipFileType, addBlanks, hashes);
logger.User(totalSize, totalSize, basePath);
}
watch.Stop();
return(true);
}
/// <summary>
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="header">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(string input, string header = null, Hash hashes = Hash.Standard, TreatAsFile asFiles = 0x00)
{
// Add safeguard if file doesn't exist
if (!File.Exists(input))
{
return(null);
}
// Get input information
var fileType = GetFileType(input);
Stream inputStream = File.OpenRead(input);
// Try to match the supplied header skipper
if (header != null)
{
SkipperMatch.Init();
var rule = SkipperMatch.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header));
// If there's a match, transform the stream before getting info
if (rule.Tests != null && rule.Tests.Count != 0)
{
// Create the output stream
MemoryStream outputStream = new MemoryStream();
// Transform the stream and get the information from it
rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true);
inputStream = outputStream;
}
}
// Get the info in the proper manner
BaseFile baseFile;
if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
baseFile = AaruFormat.Create(inputStream);
}
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
baseFile = CHDFile.Create(inputStream);
}
else
{
baseFile = GetInfo(inputStream, hashes: hashes, keepReadOpen: false);
}
// Dispose of the input stream
inputStream?.Dispose();
// Add unique data from the file
baseFile.Filename = Path.GetFileName(input);
baseFile.Date = new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss");
return(baseFile);
}
/// <summary>
/// Process a single file as a file
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles)
{
logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles);
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);
}
public override bool ProcessFeatures(Dictionary <string, Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
bool addBlankFiles = GetBoolean(features, AddBlankFilesValue);
bool addFileDates = GetBoolean(features, AddDateValue);
TreatAsFile asFiles = GetTreatAsFiles(features);
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
// Apply the specialized field removals to the cleaner
if (!addFileDates)
{
Remover.PopulateExclusionsFromList(new List <string> {
"DatItem.Date"
});
}
// Create a new DATFromDir object and process the inputs
DatFile basedat = DatFile.Create(Header);
basedat.Header.Date = DateTime.Now.ToString("yyyy-MM-dd");
// For each input directory, create a DAT
foreach (string path in Inputs)
{
if (Directory.Exists(path) || File.Exists(path))
{
// Clone the base Dat for information
DatFile datdata = DatFile.Create(basedat.Header);
// Get the base path and fill the header, if needed
string basePath = Path.GetFullPath(path);
datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path
bool success = DatTools.DatFromDir.PopulateFromDir(
datdata,
basePath,
asFiles,
skipFileType,
addBlankFiles,
hashes: includeInScan);
if (success)
{
// Perform additional processing steps
Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, false);
Filter.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata);
Remover.ApplyRemovals(datdata);
// Write out the file
Writer.Write(datdata, OutputDir);
}
else
{
Console.WriteLine();
OutputRecursive(0);
}
}
}
return(true);
}
/// <summary>
/// Check a given file for hashes, based on current settings
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">Filename of the item to be checked</param>
/// <param name="basePath">Base folder to be used in creating the DAT</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
/// <param name="skipFileType">Type of files that should be skipped</param>
/// <param name="addBlanks">True if blank items should be created for empty folders, false otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
private static void CheckFileForHashes(
DatFile datFile,
string item,
string basePath,
TreatAsFile asFiles,
SkipFileType skipFileType,
bool addBlanks,
Hash hashes)
{
// If we're in depot mode, process it separately
if (CheckDepotFile(datFile, item))
{
return;
}
// Initialize possible archive variables
BaseArchive archive = BaseArchive.Create(item);
// Process archives according to flags
if (archive != null)
{
// Set the archive flags
archive.AvailableHashes = hashes;
// Skip if we're treating archives as files and skipping files
if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File)
{
return;
}
// Skip if we're skipping archives
else if (skipFileType == SkipFileType.Archive)
{
return;
}
// Process as archive if we're not treating archives as files
else if (!asFiles.HasFlag(TreatAsFile.Archive))
{
var extracted = archive.GetChildren();
// If we have internal items to process, do so
if (extracted != null)
{
ProcessArchive(datFile, item, basePath, extracted);
}
// Now find all folders that are empty, if we are supposed to
if (addBlanks)
{
ProcessArchiveBlanks(datFile, item, basePath, archive);
}
}
// Process as file if we're treating archives as files
else
{
ProcessFile(datFile, item, basePath, hashes, asFiles);
}
}
// Process non-archives according to flags
else
{
// Skip if we're skipping files
if (skipFileType == SkipFileType.File)
{
return;
}
// Process as file
else
{
ProcessFile(datFile, item, basePath, hashes, asFiles);
}
}
}
public override void ProcessFeatures(Dictionary <string, Feature> features)
{
base.ProcessFeatures(features);
// Get a list of files from the input datfiles
var datfiles = GetList(features, DatListValue);
var datfilePaths = DirectoryExtensions.GetFilesOnly(datfiles);
// Get feature flags
TreatAsFile asFiles = GetTreatAsFiles(features);
bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue);
var splitType = GetSplitType(features);
// If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue))
{
foreach (ParentablePath datfile in datfilePaths)
{
// Parse in from the file
DatFile datdata = DatFile.Create();
datdata.Parse(datfile, int.MaxValue, keep: true);
// Perform additional processing steps
datdata.ApplyExtras(Extras);
datdata.ApplySplitting(splitType, true);
datdata.ApplyFilter(Filter);
datdata.ApplyCleaning(Cleaner);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
// If we have the depot flag, respect it
if (Header.InputDepot?.IsActive ?? false)
{
datdata.VerifyDepot(Inputs);
}
else
{
// Loop through and add the inputs to check against
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
datdata.PopulateFromDir(input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
}
datdata.VerifyGeneric(hashOnly);
}
// Now write out if there are any items left
datdata.WriteStatsToConsole();
datdata.Write(OutputDir);
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = DatFile.Create();
foreach (ParentablePath datfile in datfilePaths)
{
datdata.Parse(datfile, int.MaxValue, keep: true);
}
// Perform additional processing steps
datdata.ApplyExtras(Extras);
datdata.ApplySplitting(splitType, true);
datdata.ApplyFilter(Filter);
datdata.ApplyCleaning(Cleaner);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
watch.Stop();
// If we have the depot flag, respect it
if (Header.InputDepot?.IsActive ?? false)
{
datdata.VerifyDepot(Inputs);
}
else
{
// Loop through and add the inputs to check against
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
datdata.PopulateFromDir(input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
}
datdata.VerifyGeneric(hashOnly);
}
// Now write out if there are any items left
datdata.WriteStatsToConsole();
datdata.Write(OutputDir);
}
}
public override void ProcessFeatures(Dictionary <string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
TreatAsFile asFiles = GetTreatAsFiles(features);
bool date = GetBoolean(features, AddDateValue);
bool delete = GetBoolean(features, DeleteValue);
bool inverse = GetBoolean(features, InverseValue);
bool quickScan = GetBoolean(features, QuickValue);
bool updateDat = GetBoolean(features, UpdateDatValue);
var outputFormat = GetOutputFormat(features);
// If we have the romba flag
if (Header.OutputDepot?.IsActive == true)
{
// Update TorrentGzip output
if (outputFormat == OutputFormat.TorrentGzip)
{
outputFormat = OutputFormat.TorrentGzipRomba;
}
// Update TorrentXz output
else if (outputFormat == OutputFormat.TorrentXZ)
{
outputFormat = OutputFormat.TorrentXZRomba;
}
}
// Get a list of files from the input datfiles
var datfiles = GetList(features, DatListValue);
var datfilePaths = DirectoryExtensions.GetFilesOnly(datfiles);
// If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir
if (GetBoolean(features, IndividualValue))
{
foreach (ParentablePath datfile in datfilePaths)
{
DatFile datdata = DatFile.Create();
datdata.Parse(datfile, int.MaxValue, keep: true);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
datdata.Header.OutputDepot = Header.OutputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
// If we have the depot flag, respect it
bool success;
if (Header.InputDepot?.IsActive ?? false)
{
success = datdata.RebuildDepot(Inputs, Path.Combine(OutputDir, datdata.Header.FileName), date, delete, inverse, outputFormat);
}
else
{
success = datdata.RebuildGeneric(Inputs, Path.Combine(OutputDir, datdata.Header.FileName), quickScan, date, delete, inverse, outputFormat, asFiles);
}
// If we have a success and we're updating the DAT, write it out
if (success && updateDat)
{
datdata.Header.FileName = $"fixDAT_{Header.FileName}";
datdata.Header.Name = $"fixDAT_{Header.Name}";
datdata.Header.Description = $"fixDAT_{Header.Description}";
datdata.Items.ClearMarked();
datdata.Write(OutputDir);
}
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = DatFile.Create();
foreach (ParentablePath datfile in datfilePaths)
{
datdata.Parse(datfile, int.MaxValue, keep: true);
}
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
datdata.Header.OutputDepot = Header.OutputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
watch.Stop();
// If we have the depot flag, respect it
bool success;
if (Header.InputDepot?.IsActive ?? false)
{
success = datdata.RebuildDepot(Inputs, OutputDir, date, delete, inverse, outputFormat);
}
else
{
success = datdata.RebuildGeneric(Inputs, OutputDir, quickScan, date, delete, inverse, outputFormat, asFiles);
}
// If we have a success and we're updating the DAT, write it out
if (success && updateDat)
{
datdata.Header.FileName = $"fixDAT_{Header.FileName}";
datdata.Header.Name = $"fixDAT_{Header.Name}";
datdata.Header.Description = $"fixDAT_{Header.Description}";
datdata.Items.ClearMarked();
datdata.Write(OutputDir);
}
}
}
public override void ProcessFeatures(Dictionary <string, Library.Help.Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
bool addBlankFiles = GetBoolean(features, AddBlankFilesValue);
bool addFileDates = GetBoolean(features, AddDateValue);
TreatAsFile asFiles = GetTreatAsFiles(features);
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
var splitType = GetSplitType(features);
// Apply the specialized field removals to the cleaner
if (Cleaner.ExcludeFields == null)
{
Cleaner.ExcludeFields = new List <Field>();
}
if (!addFileDates)
{
Cleaner.ExcludeFields.Add(Field.DatItem_Date);
}
// Create a new DATFromDir object and process the inputs
DatFile basedat = DatFile.Create(Header);
basedat.Header.Date = DateTime.Now.ToString("yyyy-MM-dd");
// For each input directory, create a DAT
foreach (string path in Inputs)
{
if (Directory.Exists(path) || File.Exists(path))
{
// Clone the base Dat for information
DatFile datdata = DatFile.Create(basedat.Header);
// Get the base path and fill the header, if needed
string basePath = Path.GetFullPath(path);
datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path
bool success = datdata.PopulateFromDir(
basePath,
asFiles,
skipFileType,
addBlankFiles,
hashes: includeInScan);
if (success)
{
// Perform additional processing steps
datdata.ApplyExtras(Extras);
datdata.ApplySplitting(splitType, false);
datdata.ApplyFilter(Filter);
datdata.ApplyCleaning(Cleaner);
// Write out the file
datdata.Write(OutputDir);
}
else
{
Console.WriteLine();
OutputRecursive(0);
}
}
}
}
/// <summary>
/// Attempt to add a file to the output if it matches
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if the file was used to rebuild, false otherwise</returns>
private static bool RebuildGenericHelper(
DatFile datFile,
string file,
string outDir,
bool quickScan,
bool date,
bool inverse,
OutputFormat outputFormat,
TreatAsFile asFiles)
{
// If we somehow have a null filename, return
if (file == null)
{
return(false);
}
// Set the deletion variables
bool usedExternally = false, usedInternally = false;
// Create an empty list of BaseFile for archive entries
List <BaseFile> entries = null;
// Get the TGZ and TXZ status for later
GZipArchive tgz = new GZipArchive(file);
XZArchive txz = new XZArchive(file);
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))
{
BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles);
// Create the correct DatItem
DatItem internalDatItem;
if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
internalDatItem = new Media(internalFileInfo);
}
else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
internalDatItem = new Disk(internalFileInfo);
}
else
{
internalDatItem = new Rom(internalFileInfo);
}
usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat);
}
// Otherwise, loop through the entries and try to match
else
{
foreach (BaseFile entry in entries)
{
DatItem internalDatItem = DatItem.Create(entry);
usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */);
}
}
return(usedExternally || usedInternally);
}
/// <summary>
/// Process the DAT and find all matches in input files and folders
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="inputs">List of input files/folders to check</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="delete">True if input files should be deleted, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public static bool RebuildGeneric(
DatFile datFile,
List <string> inputs,
string outDir,
bool quickScan = false,
bool date = false,
bool delete = false,
bool inverse = false,
OutputFormat outputFormat = OutputFormat.Folder,
TreatAsFile asFiles = 0x00)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (datFile.Items.TotalCount == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return(false);
}
// Check that the output directory exists
if (!Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
outDir = Path.GetFullPath(outDir);
}
// Now we want to get forcepack flag if it's not overridden
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking != PackingFlag.None)
{
outputFormat = GetOutputFormat(datFile.Header.ForcePacking);
}
#endregion
bool success = true;
#region Rebuild from sources in order
string format = FromOutputFormat(outputFormat) ?? string.Empty;
InternalStopwatch watch = new InternalStopwatch($"Rebuilding all files to {format}");
// Now loop through all of the files in all of the inputs
foreach (string input in inputs)
{
// If the input is a file
if (File.Exists(input))
{
logger.User($"Checking file: {input}");
bool rebuilt = RebuildGenericHelper(datFile, input, outDir, quickScan, date, inverse, outputFormat, asFiles);
// If we are supposed to delete the file, do so
if (delete && rebuilt)
{
File.Delete(input);
}
}
// If the input is a directory
else if (Directory.Exists(input))
{
logger.Verbose($"Checking directory: {input}");
foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
{
logger.User($"Checking file: {file}");
bool rebuilt = RebuildGenericHelper(datFile, file, outDir, quickScan, date, inverse, outputFormat, asFiles);
// If we are supposed to delete the file, do so
if (delete && rebuilt)
{
File.Delete(input);
}
}
}
}
watch.Stop();
#endregion
return(success);
}
