public void LynxTest(int offset, byte[] content)
{
// Create the stream with the required input
var ms = new MemoryStream();
for (int i = 0; i < offset; i++)
{
ms.WriteByte(0xFF);
}
ms.Write(content);
PadAndReset(ms);
// Get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty, true);
ms.Seek(0, SeekOrigin.Begin);
// Transform the stream
var actual = new MemoryStream();
rule.TransformStream(ms, actual, false, true);
// Validate expected length
Assert.Equal(1024 - 0x40, actual.Length);
}
/// <summary>
/// Detect header skipper compliance and create an output file
/// </summary>
/// <param name="file">Name of the file to be parsed</param>
/// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param>
/// <param name="nostore">True if headers should not be stored in the database, false otherwise</param>
/// <returns>True if the output file was created, false otherwise</returns>
private bool DetectTransformStore(string file, string outDir, bool nostore)
{
// Create the output directory if it doesn't exist
if (!string.IsNullOrWhiteSpace(outDir) && !Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
}
logger.User($"\nGetting skipper information for '{file}'");
// Get the skipper rule that matches the file, if any
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(file, string.Empty);
// If we have an empty rule, return false
if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None)
{
return(false);
}
logger.User("File has a valid copier header");
// Get the header bytes from the file first
string hstr;
try
{
// Extract the header as a string for the database
using var fs = File.OpenRead(file);
byte[] hbin = new byte[(int)rule.StartOffset];
fs.Read(hbin, 0, (int)rule.StartOffset);
hstr = Utilities.ByteArrayToString(hbin);
}
catch
{
return(false);
}
// Apply the rule to the file
string newfile = (string.IsNullOrWhiteSpace(outDir) ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file)));
rule.TransformFile(file, newfile);
// If the output file doesn't exist, return false
if (!File.Exists(newfile))
{
return(false);
}
// Now add the information to the database if it's not already there
if (!nostore)
{
BaseFile baseFile = BaseFile.GetInfo(newfile, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile);
}
return(true);
}
/// <summary>
/// Retrieve file information for a single file
/// </summary>
/// <param name="input">Filename to get information from</param>
/// <param name="header">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="hashes">Hashes to include in the information</param>
/// <param name="asFiles">TreatAsFiles representing special format scanning</param>
/// <returns>Populated BaseFile object if success, empty one on error</returns>
public static BaseFile GetInfo(string input, string header = null, Hash hashes = Hash.Standard, TreatAsFile asFiles = 0x00)
{
// Add safeguard if file doesn't exist
if (!File.Exists(input))
{
return(null);
}
// Get input information
var fileType = GetFileType(input);
Stream inputStream = File.OpenRead(input);
// Try to match the supplied header skipper
if (header != null)
{
SkipperMatch.Init();
var rule = SkipperMatch.GetMatchingRule(input, Path.GetFileNameWithoutExtension(header));
// If there's a match, transform the stream before getting info
if (rule.Tests != null && rule.Tests.Count != 0)
{
// Create the output stream
MemoryStream outputStream = new MemoryStream();
// Transform the stream and get the information from it
rule.TransformStream(inputStream, outputStream, keepReadOpen: false, keepWriteOpen: true);
inputStream = outputStream;
}
}
// Get the info in the proper manner
BaseFile baseFile;
if (fileType == FileType.AaruFormat && !asFiles.HasFlag(TreatAsFile.AaruFormat))
{
baseFile = AaruFormat.Create(inputStream);
}
else if (fileType == FileType.CHD && !asFiles.HasFlag(TreatAsFile.CHD))
{
baseFile = CHDFile.Create(inputStream);
}
else
{
baseFile = GetInfo(inputStream, hashes: hashes, keepReadOpen: false);
}
// Dispose of the input stream
inputStream?.Dispose();
// Add unique data from the file
baseFile.Filename = Path.GetFileName(input);
baseFile.Date = new FileInfo(input).LastWriteTime.ToString("yyyy/MM/dd HH:mm:ss");
return(baseFile);
}
public void N64Test(byte[] content)
{
// Create the stream with the required input
var ms = new MemoryStream();
ms.Write(content);
ms.Seek(0, SeekOrigin.Begin);
// Check that we get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty);
Assert.NotNull(rule);
Assert.Equal("n64", rule.SourceFile);
}
public void SPCTest()
{
// Create the stream with the required input
var ms = new MemoryStream();
ms.Write(new byte[] { 0x53, 0x4E, 0x45, 0x53, 0x2D, 0x53, 0x50, 0x43 });
ms.Seek(0, SeekOrigin.Begin);
// Check that we get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty);
Assert.NotNull(rule);
Assert.Equal("spc", rule.SourceFile);
}
public void PCETestInternal()
{
// Create the stream with the required input
var ms = new MemoryStream();
ms.Write(new byte[] { 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0x02 });
ms.Seek(0, SeekOrigin.Begin);
// Check that we get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty);
Assert.NotNull(rule);
Assert.Equal("pce", rule.SourceFile);
}
public void LynxTest(int offset, byte[] content)
{
// Create the stream with the required input
var ms = new MemoryStream();
for (int i = 0; i < offset; i++)
{
ms.WriteByte(0xFF);
}
ms.Write(content);
ms.Seek(0, SeekOrigin.Begin);
// Check that we get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty);
Assert.NotNull(rule);
Assert.Equal("lynx", rule.SourceFile);
}
public void SPCTest()
{
// Create the stream with the required input
var ms = new MemoryStream();
ms.Write(new byte[] { 0x53, 0x4E, 0x45, 0x53, 0x2D, 0x53, 0x50, 0x43 });
PadAndReset(ms);
// Get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty, true);
ms.Seek(0, SeekOrigin.Begin);
// Transform the stream
var actual = new MemoryStream();
rule.TransformStream(ms, actual, false, true);
// Validate expected length
Assert.Equal(1024 - 0x100, actual.Length);
}
public void PSIDTest(byte[] content, int startOffset)
{
// Create the stream with the required input
var ms = new MemoryStream();
ms.Write(content);
PadAndReset(ms);
// Get a match
var rule = SkipperMatch.GetMatchingRule(ms, string.Empty, true);
ms.Seek(0, SeekOrigin.Begin);
// Transform the stream
var actual = new MemoryStream();
rule.TransformStream(ms, actual, false, true);
// Validate expected length
Assert.Equal(1024 - startOffset, actual.Length);
}
/// <summary>
/// Find duplicates and rebuild individual files to output
/// </summary>
/// <param name="datFile">Current DatFile object to rebuild from</param>
/// <param name="datItem">Information for the current file to rebuild from</param>
/// <param name="file">Name of the file to process</param>
/// <param name="outDir">Output directory to use to build to</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
/// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
/// <param name="outputFormat">Output format that files should be written to</param>
/// <param name="isZip">True if the input file is an archive, false if the file is TGZ/TXZ, null otherwise</param>
/// <returns>True if the file was able to be rebuilt, false otherwise</returns>
private static bool RebuildIndividualFile(
DatFile datFile,
DatItem datItem,
string file,
string outDir,
bool date,
bool inverse,
OutputFormat outputFormat,
bool?isZip = null)
{
// Set the initial output value
bool rebuilt = false;
// If the DatItem is a Disk or Media, force rebuilding to a folder except if TGZ or TXZ
if ((datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.Media) &&
!(outputFormat == OutputFormat.TorrentGzip || outputFormat == OutputFormat.TorrentGzipRomba) &&
!(outputFormat == OutputFormat.TorrentXZ || outputFormat == OutputFormat.TorrentXZRomba))
{
outputFormat = OutputFormat.Folder;
}
// If we have a Disk or Media, change it into a Rom for later use
if (datItem.ItemType == ItemType.Disk)
{
datItem = (datItem as Disk).ConvertToRom();
}
else if (datItem.ItemType == ItemType.Media)
{
datItem = (datItem as Media).ConvertToRom();
}
// Prepopluate a key string
string crc = (datItem as Rom).CRC ?? string.Empty;
// Try to get the stream for the file
if (!GetFileStream(datItem, file, isZip, out Stream fileStream))
{
return(false);
}
// If either we have duplicates or we're filtering
if (ShouldRebuild(datFile, datItem, fileStream, inverse, out ConcurrentList <DatItem> dupes))
{
// If we have a very specific TGZ->TGZ case, just copy it accordingly
if (RebuildTorrentGzip(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
// If we have a very specific TXZ->TXZ case, just copy it accordingly
if (RebuildTorrentXz(datFile, datItem, file, outDir, outputFormat, isZip))
{
return(true);
}
logger.User($"{(inverse ? "No matches" : "Matches")} found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Special case for partial packing mode
bool shouldCheck = false;
if (outputFormat == OutputFormat.Folder && datFile.Header.ForcePacking == PackingFlag.Partial)
{
shouldCheck = true;
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false);
}
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// If we should check for the items in the machine
if (shouldCheck && datFile.Items[item.Machine.Name].Count > 1)
{
outputFormat = OutputFormat.Folder;
}
else if (shouldCheck && datFile.Items[item.Machine.Name].Count == 1)
{
outputFormat = OutputFormat.ParentFolder;
}
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
outputArchive.Write(fileStream, outDir, (item as Rom).ConvertToBaseFile());
}
// Close the input stream
fileStream?.Dispose();
}
// Now we want to take care of headers, if applicable
if (datFile.Header.HeaderSkipper != null)
{
// Check to see if we have a matching header first
SkipperMatch.Init();
SkipperRule rule = SkipperMatch.GetMatchingRule(fileStream, Path.GetFileNameWithoutExtension(datFile.Header.HeaderSkipper));
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
MemoryStream transformStream = new MemoryStream();
if (rule.TransformStream(fileStream, transformStream, keepReadOpen: true, keepWriteOpen: true))
{
// Get the file informations that we will be using
Rom headerless = new Rom(BaseFile.GetInfo(transformStream, keepReadOpen: true));
// If we have duplicates and we're not filtering
if (ShouldRebuild(datFile, headerless, transformStream, false, out dupes))
{
logger.User($"Headerless matches found for '{Path.GetFileName(datItem.GetName() ?? datItem.ItemType.ToString())}', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (DatItem item in dupes)
{
// Create a headered item to use as well
datItem.CopyMachineInformation(item);
datItem.SetName($"{datItem.GetName()}_{crc}");
// Get the output archive, if possible
Folder outputArchive = GetPreconfiguredFolder(datFile, date, outputFormat);
// Now rebuild to the output file
bool eitherSuccess = false;
eitherSuccess |= outputArchive.Write(transformStream, outDir, (item as Rom).ConvertToBaseFile());
eitherSuccess |= outputArchive.Write(fileStream, outDir, (datItem as Rom).ConvertToBaseFile());
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
}
}
}
// Dispose of the stream
transformStream?.Dispose();
}
// Dispose of the stream
fileStream?.Dispose();
}
return(rebuilt);
}