/// <summary>
/// Detect and replace header(s) to the given file
/// </summary>
/// <param name="file">Name of the file to be parsed</param>
/// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param>
/// <returns>True if a header was found and appended, false otherwise</returns>
public bool RestoreHeader(string file, string outDir)
{
// Create the output directory if it doesn't exist
if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
}
// First, get the SHA-1 hash of the file
BaseFile baseFile = BaseFile.GetInfo(file, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
// Retrieve a list of all related headers from the database
List <string> headers = RetrieveHeadersFromDatabase(Utilities.ByteArrayToString(baseFile.SHA1));
// If we have nothing retrieved, we return false
if (headers.Count == 0)
{
return(false);
}
// Now loop through and create the reheadered files, if possible
for (int i = 0; i < headers.Count; i++)
{
string outputFile = (string.IsNullOrWhiteSpace(outDir) ? $"{Path.GetFullPath(file)}.new" : Path.Combine(outDir, Path.GetFileName(file))) + i;
logger.User($"Creating reheadered file: {outputFile}");
AppendBytes(file, outputFile, Utilities.StringToByteArray(headers[i]), null);
logger.User("Reheadered file created!");
}
return(true);
}
/// <summary>
/// Process a single file as a file
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="item">File to be added</param>
/// <param name="basePath">Path the represents the parent directory</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
private static void ProcessFile(DatFile datFile, string item, string basePath, Hash hashes, TreatAsFile asFiles)
{
logger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
BaseFile baseFile = BaseFile.GetInfo(item, header: datFile.Header.HeaderSkipper, hashes: hashes, asFiles: asFiles);
DatItem datItem = DatItem.Create(baseFile);
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);
}
/// <summary>
/// Detect header skipper compliance and create an output file
/// </summary>
/// <param name="file">Name of the file to be parsed</param>
/// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param>
/// <param name="nostore">True if headers should not be stored in the database, false otherwise</param>
/// <returns>True if the output file was created, false otherwise</returns>
private bool DetectTransformStore(string file, string outDir, bool nostore)
{
// Create the output directory if it doesn't exist
if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
}
logger.User($"\nGetting skipper information for '{file}'");
// Get the skipper rule that matches the file, if any
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(file, string.Empty);
// If we have an empty rule, return false
if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None)
{
return(false);
}
logger.User("File has a valid copier header");
// Get the header bytes from the file first
string hstr;
try
{
// Extract the header as a string for the database
using var fs = File.OpenRead(file);
byte[] hbin = new byte[(int)rule.StartOffset];
fs.Read(hbin, 0, (int)rule.StartOffset);
hstr = Utilities.ByteArrayToString(hbin);
}
catch
{
return(false);
}
// Apply the rule to the file
string newfile = (string.IsNullOrWhiteSpace(outDir) ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file)));
rule.TransformFile(file, newfile);
// If the output file doesn't exist, return false
if (!File.Exists(newfile))
{
return(false);
}
// Now add the information to the database if it's not already there
if (!nostore)
{
BaseFile baseFile = BaseFile.GetInfo(newfile, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile);
}
return(true);
}
public void GetInfoNormalFileTest()
{
// Get the path to the test data
string filename = Path.Combine(Environment.CurrentDirectory, "TestData", "file-to-hash.bin");
// Set all of the expected hash values
string expectedCrc = "ba02a660";
string expectedMd5 = "b722871eaa950016296184d026c5dec9";
string expectedSha1 = "eea1ee2d801d830c4bdad4df3c8da6f9f52d1a9f";
string expectedSha256 = "fdb02dee8c319c52087382c45f099c90d0b6cc824850aff28c1bfb2884b7b855";
string expectedSha384 = "e276c49618fff25bc1fe2e0659cd0ef0e7c1186563b063e07c52323b9899f3ce9b091be04d6208444b3ef1265e879074";
string expectedSha512 = "15d69514eb628c2403e945a7cafd1d27e557f6e336c69b63ea17e7ed9d256cc374ee662f09305836d6de37fdae59d83883b982aa8446e4ff26346b6b6b50b240";
string expectedSpamSum = "3:hMCPQCE6AFQxWyENFACBE+rW6Tj7SMQmKozr9MVERk:hZRdxZENFs+rPSromek";
// Get the BaseFile generated from the path
var baseFile = BaseFile.GetInfo(filename, hashes: Hash.All);
// Extract all the hashes to string
string actualCrc = Utilities.ByteArrayToString(baseFile.CRC);
string actualMd5 = Utilities.ByteArrayToString(baseFile.MD5);
string actualSha1 = Utilities.ByteArrayToString(baseFile.SHA1);
string actualSha256 = Utilities.ByteArrayToString(baseFile.SHA256);
string actualSha384 = Utilities.ByteArrayToString(baseFile.SHA384);
string actualSha512 = Utilities.ByteArrayToString(baseFile.SHA512);
string actualSpamSum = Encoding.UTF8.GetString(baseFile.SpamSum);
// Verify all of the hashes match
Assert.Equal(expectedCrc, actualCrc);
Assert.Equal(expectedMd5, actualMd5);
Assert.Equal(expectedSha1, actualSha1);
Assert.Equal(expectedSha256, actualSha256);
Assert.Equal(expectedSha384, actualSha384);
Assert.Equal(expectedSha512, actualSha512);
Assert.Equal(expectedSpamSum, actualSpamSum);
// Convert to Rom for sanity check
var rom = new Rom(baseFile);
// Verify all of the hashes match
Assert.Equal(expectedCrc, rom.CRC);
Assert.Equal(expectedMd5, rom.MD5);
Assert.Equal(expectedSha1, rom.SHA1);
Assert.Equal(expectedSha256, rom.SHA256);
Assert.Equal(expectedSha384, rom.SHA384);
Assert.Equal(expectedSha512, rom.SHA512);
Assert.Equal(expectedSpamSum, rom.SpamSum);
}
/// <summary>
/// Get the rebuild state for a given item
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="stream">Stream representing the input file</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="dupes">Output list of duplicate items to rebuild to</param>
/// <returns>True if the item should be rebuilt, false otherwise</returns>
private static bool ShouldRebuild(DatFile datFile, DatItem datItem, Stream stream, bool inverse, out ConcurrentList <DatItem> dupes)
{
// Find if the file has duplicates in the DAT
dupes = datFile.Items.GetDuplicates(datItem);
bool hasDuplicates = dupes.Count > 0;
// If we have duplicates but we're filtering
if (hasDuplicates && inverse)
{
return(false);
}
// If we have duplicates without filtering
else if (hasDuplicates && !inverse)
{
return(true);
}
// If we have no duplicates and we're filtering
else if (!hasDuplicates && inverse)
{
string machinename = null;
// Get the item from the current file
Rom item = new Rom(BaseFile.GetInfo(stream, keepReadOpen: true));
item.Machine.Name = Path.GetFileNameWithoutExtension(item.Name);
item.Machine.Description = Path.GetFileNameWithoutExtension(item.Name);
// If we are coming from an archive, set the correct machine name
if (machinename != null)
{
item.Machine.Name = machinename;
item.Machine.Description = machinename;
}
dupes.Add(item);
return(true);
}
// If we have no duplicates and we're not filtering
else
{
return(false);
}
}
/// <summary>
/// Gets all valid DATs that match in the DAT root
/// </summary>
/// <param name="inputs">List of input strings to check for, presumably file names</param>
/// <returns>Dictionary of hash/full path for each of the valid DATs</returns>
internal Dictionary <string, string> GetValidDats(List <string> inputs)
{
// Get a dictionary of filenames that actually exist in the DATRoot, logging which ones are not
List <string> datRootDats = Directory.EnumerateFiles(_dats, "*", SearchOption.AllDirectories).ToList();
List <string> lowerCaseDats = datRootDats.ConvertAll(i => Path.GetFileName(i).ToLowerInvariant());
Dictionary <string, string> foundDats = new Dictionary <string, string>();
foreach (string input in inputs)
{
if (lowerCaseDats.Contains(input.ToLowerInvariant()))
{
string fullpath = Path.GetFullPath(datRootDats[lowerCaseDats.IndexOf(input.ToLowerInvariant())]);
string sha1 = Utilities.ByteArrayToString(BaseFile.GetInfo(fullpath, hashes: Hash.SHA1).SHA1);
foundDats.Add(sha1, fullpath);
}
else
{
logger.Warning($"The file '{input}' could not be found in the DAT root");
}
}
return(foundDats);
}
/// <summary>
/// Find duplicates and rebuild individual files to output
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
/// <returns>True if the file was able to be rebuilt, false otherwise</returns>
private static bool RebuildIndividualFile(
DatFile datFile,
DatItem datItem,
string file,
string outDir,
bool date,
bool inverse,
OutputFormat outputFormat,
bool?isZip = null)
{
// Set the initial output value
bool rebuilt = false;
// If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) &&
!(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) &&
!(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
outputFormat = OutputFormat.Folder;
}
// If we have a Disk or Media, change it into a Rom for later use
if (datItem.ItemType == ItemType.Disk)
{
datItem = (datItem as Disk).ConvertToRom();
}
else if (datItem.ItemType == ItemType.Media)
{
datItem = (datItem as Media).ConvertToRom();
}
// Prepopluate a key string
string crc = (datItem as Rom).CRC ?? string.Empty;
// Try to get the stream for the file
if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
{
return(false);
}
// If either we have duplicates or we're filtering
if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes))
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
// If we have a very specific TXZ->TXZ case, just copy it accordingly
if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Special case for partial packing mode
bool shouldCheck = false;
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
{
shouldCheck = true;
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false);
}
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// If we should check for the items in the machine
if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
{
outputFormat = OutputFormat.Folder;
}
else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
{
outputFormat = OutputFormat.ParentFolder;
}
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
}
// Close the input stream
fileStream?.Dispose();
}
// Now we want to take care of headers, if applicable
if (datFile.Header.HeaderSkipper != null)
{
// Check to see if we have a matching header first
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
MemoryStream transformStream = new MemoryStream();
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
{
logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// Create a headered item to use as well
datItem.CopyMachineInformation(item);
datItem.SetName($"{datItem.GetName()}_{crc}");
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
bool eitherSuccess = false;
eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
}
}
}
// Dispose of the stream
transformStream?.Dispose();
}
// Dispose of the stream
fileStream?.Dispose();
}
return(rebuilt);
}
/// <summary>
/// Attempt to add a file to the output if it matches
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>True if the file was used to rebuild, false otherwise</returns>
private static bool RebuildGenericHelper(
DatFile datFile,
string file,
string outDir,
bool quickScan,
bool date,
bool inverse,
OutputFormat outputFormat,
TreatAsFile asFiles)
{
// If we somehow have a null filename, return
if (file == null)
{
return(false);
}
// Set the deletion variables
bool usedExternally = false, usedInternally = false;
// Create an empty list of BaseFile for archive entries
List <BaseFile> entries = null;
// Get the TGZ and TXZ status for later
GZipArchive tgz = new GZipArchive(file);
XZArchive txz = new XZArchive(file);
bool isSingleTorrent = tgz.IsTorrent() || txz.IsTorrent();
// Get the base archive first
BaseArchive archive = BaseArchive.Create(file);
// Now get all extracted items from the archive
if (archive != null)
{
archive.AvailableHashes = quickScan ? Hash.CRC : Hash.Standard;
entries = archive.GetChildren();
}
// If the entries list is null, we encountered an error or have a file and should scan externally
if (entries == null && File.Exists(file))
{
BaseFile internalFileInfo = BaseFile.GetInfo(file, asFiles: asFiles);
// Create the correct DatItem
DatItem internalDatItem;
if (internalFileInfo.Type == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
internalDatItem = new Media(internalFileInfo);
}
else if (internalFileInfo.Type == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
internalDatItem = new Disk(internalFileInfo);
}
else
{
internalDatItem = new Rom(internalFileInfo);
}
usedExternally = RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat);
}
// Otherwise, loop through the entries and try to match
else
{
foreach (BaseFile entry in entries)
{
DatItem internalDatItem = DatItem.Create(entry);
usedInternally |= RebuildIndividualFile(datFile, internalDatItem, file, outDir, date, inverse, outputFormat, !isSingleTorrent /* isZip */);
}
}
return(usedExternally || usedInternally);
}