public override bool ProcessFeatures(Dictionary <string, Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string source = GetString(features, SourceStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
outdat.Ensure(create: true);
// Check that all required directories exist
if (!Directory.Exists(source))
{
logger.Error($"File '{source}' does not exist!");
return(false);
}
// Create and write the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name;
datfile.Header.Description = description;
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
Writer.Write(datfile, outdat);
return(true);
}
/// <summary>
/// Output duplicate item diff
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static DatFile DiffDuplicates(DatFile datFile, List <ParentablePath> inputs)
{
InternalStopwatch watch = new InternalStopwatch("Initializing duplicate DAT");
// Fill in any information not in the base DAT
if (string.IsNullOrWhiteSpace(datFile.Header.FileName))
{
datFile.Header.FileName = "All DATs";
}
if (string.IsNullOrWhiteSpace(datFile.Header.Name))
{
datFile.Header.Name = "datFile.All DATs";
}
if (string.IsNullOrWhiteSpace(datFile.Header.Description))
{
datFile.Header.Description = "datFile.All DATs";
}
string post = " (Duplicates)";
DatFile dupeData = DatFile.Create(datFile.Header);
dupeData.Header.FileName += post;
dupeData.Header.Name += post;
dupeData.Header.Description += post;
dupeData.Items = new ItemDictionary();
watch.Stop();
// Now, loop through the dictionary and populate the correct DATs
watch.Start("Populating duplicate DAT");
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
{
return;
}
// Loop through and add the items correctly
foreach (DatItem item in items)
{
if (item.DupeType.HasFlag(DupeType.External))
{
DatItem newrom = item.Clone() as DatItem;
newrom.Machine.Name += $" ({Path.GetFileNameWithoutExtension(inputs[item.Source.Index].CurrentPath)})";
dupeData.Items.Add(key, newrom);
}
}
});
watch.Stop();
return(dupeData);
}
/// <summary>
/// Split a DAT by type of DatItem
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <returns>Dictionary of ItemType to DatFile mappings</returns>
public static Dictionary <ItemType, DatFile> SplitByType(DatFile datFile)
{
// Create each of the respective output DATs
InternalStopwatch watch = new InternalStopwatch($"Splitting DAT by item type");
// Create the set of type-to-dat mappings
Dictionary <ItemType, DatFile> typeDats = new Dictionary <ItemType, DatFile>();
// We only care about a subset of types
List <ItemType> outputTypes = new List <ItemType>
{
ItemType.Disk,
ItemType.Media,
ItemType.Rom,
ItemType.Sample,
};
// Setup all of the DatFiles
foreach (ItemType itemType in outputTypes)
{
typeDats[itemType] = DatFile.Create(datFile.Header.CloneStandard());
typeDats[itemType].Header.FileName += $" ({itemType})";
typeDats[itemType].Header.Name += $" ({itemType})";
typeDats[itemType].Header.Description += $" ({itemType})";
}
// Now populate each of the DAT objects in turn
Parallel.ForEach(outputTypes, Globals.ParallelOptions, itemType =>
{
FillWithItemType(datFile, typeDats[itemType], itemType);
});
watch.Stop();
return(typeDats);
}
/// <summary>
/// Populate from multiple paths while returning the invividual headers
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">Paths to DATs to parse</param>
/// <returns>List of DatHeader objects representing headers</returns>
public static List <DatHeader> PopulateUserData(DatFile datFile, List <ParentablePath> inputs)
{
DatFile[] datFiles = new DatFile[inputs.Count];
InternalStopwatch watch = new InternalStopwatch("Processing individual DATs");
// Parse all of the DATs into their own DatFiles in the array
Parallel.For(0, inputs.Count, Globals.ParallelOptions, i =>
{
var input = inputs[i];
logger.User($"Adding DAT: {input.CurrentPath}");
datFiles[i] = DatFile.Create(datFile.Header.CloneFiltering());
Parser.ParseInto(datFiles[i], input, i, keep: true);
});
watch.Stop();
watch.Start("Populating internal DAT");
for (int i = 0; i < inputs.Count; i++)
{
AddFromExisting(datFile, datFiles[i], true);
}
watch.Stop();
return(datFiles.Select(d => d.Header).ToList());
}
public override void ProcessFeatures(Dictionary <string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string source = GetString(features, SourceStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
DirectoryExtensions.Ensure(outdat, create: true);
// Check that all required directories exist
if (!Directory.Exists(source))
{
logger.Error($"File '{source}' does not exist!");
return;
}
// Create and write the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name;
datfile.Header.Description = description;
datfile.PopulateFromDir(source, asFiles: TreatAsFile.NonArchive);
datfile.ApplyCleaning(new Cleaner()
{
ExcludeFields = Hash.DeepHashes.AsFields()
});
datfile.Write(outdat);
}
/// <summary>
/// Create a DatFile and parse a file into it
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static DatFile CreateAndParse(string filename, bool statsOnly = false, bool throwOnError = false)
{
DatFile datFile = DatFile.Create();
ParseInto(datFile, new ParentablePath(filename), statsOnly: statsOnly, throwOnError: throwOnError);
return(datFile);
}
/// <summary>
/// Output cascading diffs
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="datHeaders">Dat headers used optionally</param>
/// <returns>List of DatFiles representing the individually indexed items</returns>
public static List <DatFile> DiffCascade(DatFile datFile, List <DatHeader> datHeaders)
{
// Create a list of DatData objects representing output files
List <DatFile> outDats = new List <DatFile>();
// Ensure the current DatFile is sorted optimally
datFile.Items.BucketBy(ItemKey.CRC, DedupeType.None);
// Loop through each of the inputs and get or create a new DatData object
InternalStopwatch watch = new InternalStopwatch("Initializing and filling all output DATs");
// Create the DatFiles from the set of headers
DatFile[] outDatsArray = new DatFile[datHeaders.Count];
Parallel.For(0, datHeaders.Count, Globals.ParallelOptions, j =>
{
DatFile diffData = DatFile.Create(datHeaders[j]);
diffData.Items = new ItemDictionary();
FillWithSourceIndex(datFile, diffData, j);
outDatsArray[j] = diffData;
});
outDats = outDatsArray.ToList();
watch.Stop();
return(outDats);
}
/// <summary>
/// Split a DAT by input extensions
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="extA">List of extensions to split on (first DAT)</param>
/// <param name="extB">List of extensions to split on (second DAT)</param>
/// <returns>Extension Set A and Extension Set B DatFiles</returns>
public static (DatFile extADat, DatFile extBDat) SplitByExtension(DatFile datFile, List <string> extA, List <string> extB)
{
// If roms is empty, return false
if (datFile.Items.TotalCount == 0)
{
return(null, null);
}
InternalStopwatch watch = new InternalStopwatch($"Splitting DAT by extension");
// Make sure all of the extensions don't have a dot at the beginning
var newExtA = extA.Select(s => s.TrimStart('.').ToLowerInvariant());
string newExtAString = string.Join(",", newExtA);
var newExtB = extB.Select(s => s.TrimStart('.').ToLowerInvariant());
string newExtBString = string.Join(",", newExtB);
// Set all of the appropriate outputs for each of the subsets
DatFile extADat = DatFile.Create(datFile.Header.CloneStandard());
extADat.Header.FileName += $" ({newExtAString})";
extADat.Header.Name += $" ({newExtAString})";
extADat.Header.Description += $" ({newExtAString})";
DatFile extBDat = DatFile.Create(datFile.Header.CloneStandard());
extBDat.Header.FileName += $" ({newExtBString})";
extBDat.Header.Name += $" ({newExtBString})";
extBDat.Header.Description += $" ({newExtBString})";
// Now separate the roms accordingly
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
if (newExtA.Contains((item.GetName() ?? string.Empty).GetNormalizedExtension()))
{
extADat.Items.Add(key, item);
}
else if (newExtB.Contains((item.GetName() ?? string.Empty).GetNormalizedExtension()))
{
extBDat.Items.Add(key, item);
}
else
{
extADat.Items.Add(key, item);
extBDat.Items.Add(key, item);
}
}
});
// Then return both DatFiles
watch.Stop();
return(extADat, extBDat);
}
/// <summary>
/// Split a DAT by size of Rom
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="radix">Long value representing the split point</param>
/// <returns>Less Than and Greater Than DatFiles</returns>
public static (DatFile lessThan, DatFile greaterThan) SplitBySize(DatFile datFile, long radix)
{
// Create each of the respective output DATs
InternalStopwatch watch = new InternalStopwatch($"Splitting DAT by size");
DatFile lessThan = DatFile.Create(datFile.Header.CloneStandard());
lessThan.Header.FileName += $" (less than {radix})";
lessThan.Header.Name += $" (less than {radix})";
lessThan.Header.Description += $" (less than {radix})";
DatFile greaterThan = DatFile.Create(datFile.Header.CloneStandard());
greaterThan.Header.FileName += $" (equal-greater than {radix})";
greaterThan.Header.Name += $" (equal-greater than {radix})";
greaterThan.Header.Description += $" (equal-greater than {radix})";
// Now populate each of the DAT objects in turn
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the file is not a Rom, it automatically goes in the "lesser" dat
if (item.ItemType != ItemType.Rom)
{
lessThan.Items.Add(key, item);
}
// If the file is a Rom and has no size, put it in the "lesser" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size == null)
{
lessThan.Items.Add(key, item);
}
// If the file is a Rom and less than the radix, put it in the "lesser" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size < radix)
{
lessThan.Items.Add(key, item);
}
// If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat
else if (item.ItemType == ItemType.Rom && (item as Rom).Size >= radix)
{
greaterThan.Items.Add(key, item);
}
}
});
// Then return both DatFiles
watch.Stop();
return(lessThan, greaterThan);
}
/// <summary>
/// Parse a DAT and return all found games and roms within
/// </summary>
/// <param name="datFile">Current DatFile object to add to</param>
/// <param name="input">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="keepext">True if original extension should be kept, false otherwise (default)</param>
/// <param name="quotes">True if quotes are assumed in supported types (default), false otherwise</param>
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public static void ParseInto(
DatFile datFile,
ParentablePath input,
int indexId = 0,
bool keep = false,
bool keepext = false,
bool quotes = true,
bool statsOnly = false,
bool throwOnError = true)
{
// Get the current path from the filename
string currentPath = input.CurrentPath;
// Check the file extension first as a safeguard
if (!Utilities.HasValidDatExtension(currentPath))
{
return;
}
// If the output filename isn't set already, get the internal filename
datFile.Header.FileName = string.IsNullOrWhiteSpace(datFile.Header.FileName)
? (keepext
? Path.GetFileName(currentPath)
: Path.GetFileNameWithoutExtension(currentPath))
: datFile.Header.FileName;
// If the output type isn't set already, get the internal output type
DatFormat currentPathFormat = GetDatFormat(currentPath);
datFile.Header.DatFormat = datFile.Header.DatFormat == 0 ? currentPathFormat : datFile.Header.DatFormat;
datFile.Items.SetBucketedBy(ItemKey.CRC); // Setting this because it can reduce issues later
InternalStopwatch watch = new InternalStopwatch($"Parsing '{currentPath}' into internal DAT");
// Now parse the correct type of DAT
try
{
var parsingDatFile = DatFile.Create(currentPathFormat, datFile, quotes);
parsingDatFile?.ParseFile(currentPath, indexId, keep, statsOnly: statsOnly, throwOnError: throwOnError);
}
catch (Exception ex) when(!throwOnError)
{
logger.Error(ex, $"Error with file '{currentPath}'");
}
watch.Stop();
}
public override bool ProcessFeatures(Dictionary <string, Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string newdat = GetString(features, NewStringValue);
string olddat = GetString(features, OldStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
outdat.Ensure(create: true);
// Check that all required files exist
if (!File.Exists(olddat))
{
logger.Error($"File '{olddat}' does not exist!");
return(false);
}
if (!File.Exists(newdat))
{
logger.Error($"File '{newdat}' does not exist!");
return(false);
}
// Create the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = name;
datfile.Header.Description = description;
Parser.ParseInto(datfile, olddat);
// Diff against the new datfile
DatFile intDat = Parser.CreateAndParse(newdat);
DatFileTool.DiffAgainst(datfile, intDat, false);
Writer.Write(intDat, outdat);
return(true);
}
/// <summary>
/// Split a SuperDAT by lowest available directory level
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <param name="shortname">True if short names should be used, false otherwise</param>
/// <param name="basedat">True if original filenames should be used as the base for output filename, false otherwise</param>
/// <returns>True if split succeeded, false otherwise</returns>
public static bool SplitByLevel(DatFile datFile, string outDir, bool shortname, bool basedat)
{
InternalStopwatch watch = new InternalStopwatch($"Splitting DAT by level");
// First, bucket by games so that we can do the right thing
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None, lower: false, norename: true);
// Create a temporary DAT to add things to
DatFile tempDat = DatFile.Create(datFile.Header);
tempDat.Header.Name = null;
// Sort the input keys
List <string> keys = datFile.Items.Keys.ToList();
keys.Sort(SplitByLevelSort);
// Then, we loop over the games
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
{
// Here, the key is the name of the game to be used for comparison
if (tempDat.Header.Name != null && tempDat.Header.Name != Path.GetDirectoryName(key))
{
// Reset the DAT for the next items
tempDat = DatFile.Create(datFile.Header);
tempDat.Header.Name = null;
}
// Clean the input list and set all games to be pathless
ConcurrentList <DatItem> items = datFile.Items[key];
items.ForEach(item => item.Machine.Name = Path.GetFileName(item.Machine.Name));
items.ForEach(item => item.Machine.Description = Path.GetFileName(item.Machine.Description));
// Now add the game to the output DAT
tempDat.Items.AddRange(key, items);
// Then set the DAT name to be the parent directory name
tempDat.Header.Name = Path.GetDirectoryName(key);
});
watch.Stop();
return(true);
}
public override void ProcessFeatures(Dictionary <string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string newdat = GetString(features, NewStringValue);
string olddat = GetString(features, OldStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
DirectoryExtensions.Ensure(outdat, create: true);
// Check that all required files exist
if (!File.Exists(olddat))
{
logger.Error($"File '{olddat}' does not exist!");
return;
}
if (!File.Exists(newdat))
{
logger.Error($"File '{newdat}' does not exist!");
return;
}
// Create the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = name;
datfile.Header.Description = description;
datfile.Parse(olddat);
// Diff against the new datfile
DatFile intDat = DatFile.CreateAndParse(newdat);
datfile.DiffAgainst(intDat, false);
intDat.Write(outdat);
}
/// <summary>
/// Reset the current state
/// </summary>
public void Reset()
{
this.Index = 0;
this.DatFile = DatFile.Create();
this.OutputDirectory = null;
}
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get the archive scanning level
// TODO: Remove usage
int sevenzip = GetInt32(features, Include7ZipsInt32Value);
int gz = GetInt32(features, IncludeGZipsInt32Value);
int zip = GetInt32(features, IncludeZipsInt32Value);
// Get feature flags
bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
// First we want to get just all directories from the inputs
List <string> onlyDirs = new List <string>();
foreach (string input in Inputs)
{
if (Directory.Exists(input))
{
onlyDirs.Add(Path.GetFullPath(input));
}
}
// Then process all of the input directories into an internal DAT
DatFile df = DatFile.Create();
foreach (string dir in onlyDirs)
{
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: Hash.Standard);
}
// Create an empty Dat for files that need to be rebuilt
DatFile need = DatFile.Create();
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now that we have the Dats, add the files to the database
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
foreach (string key in df.Items.Keys)
{
ConcurrentList <DatItem> datItems = df.Items[key];
foreach (Rom rom in datItems)
{
// If we care about if the file exists, check the databse first
if (onlyNeeded && !noDb)
{
string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"
+ $" WHERE crcsha1.crc=\"{rom.CRC}\""
+ $" OR md5sha1.md5=\"{rom.MD5}\""
+ $" OR md5sha1.sha1=\"{rom.SHA1}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
// Add to the queries
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += $" (\"{rom.CRC}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5query += $" (\"{rom.MD5}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
// Add to the Dat
need.Items.Add(key, rom);
}
}
// Otherwise, just add the file to the list
else
{
// Add to the queries
if (!noDb)
{
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += $" (\"{rom.CRC}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5query += $" (\"{rom.MD5}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
}
// Add to the Dat
need.Items.Add(key, rom);
}
}
}
// Now run the queries, if they're populated
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
// Create the sorting object to use and rebuild the needed files
Rebuilder.RebuildGeneric(
need,
onlyDirs,
outDir: _depots.Keys.ToList()[0],
outputFormat: OutputFormat.TorrentGzipRomba,
asFiles: TreatAsFile.NonArchive);
return(true);
}
public override bool ProcessFeatures(Dictionary <string, Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get the splitting mode
SplittingMode splittingMode = GetSplittingMode(features);
if (splittingMode == SplittingMode.None)
{
logger.Error("No valid splitting mode found!");
return(false);
}
// Get only files from the inputs
List <ParentablePath> files = PathTool.GetFilesOnly(Inputs, appendparent: true);
// Loop over the input files
foreach (ParentablePath file in files)
{
// Create and fill the new DAT
DatFile internalDat = DatFile.Create(Header);
Parser.ParseInto(internalDat, file);
// Get the output directory
OutputDir = OutputDir.Ensure();
OutputDir = file.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
// Extension splitting
if (splittingMode.HasFlag(SplittingMode.Extension))
{
(DatFile extADat, DatFile extBDat) = DatTools.Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue));
InternalStopwatch watch = new InternalStopwatch("Outputting extension-split DATs");
// Output both possible DatFiles
Writer.Write(extADat, OutputDir);
Writer.Write(extBDat, OutputDir);
watch.Stop();
}
// Hash splitting
if (splittingMode.HasFlag(SplittingMode.Hash))
{
Dictionary <DatItemField, DatFile> typeDats = DatTools.Splitter.SplitByHash(internalDat);
InternalStopwatch watch = new InternalStopwatch("Outputting hash-split DATs");
// Loop through each type DatFile
Parallel.ForEach(typeDats.Keys, Globals.ParallelOptions, itemType =>
{
Writer.Write(typeDats[itemType], OutputDir);
});
watch.Stop();
}
// Level splitting
if (splittingMode.HasFlag(SplittingMode.Level))
{
logger.Warning("This feature is not implemented: level-split");
DatTools.Splitter.SplitByLevel(
internalDat,
OutputDir,
GetBoolean(features, ShortValue),
GetBoolean(features, BaseValue));
}
// Size splitting
if (splittingMode.HasFlag(SplittingMode.Size))
{
(DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value));
InternalStopwatch watch = new InternalStopwatch("Outputting size-split DATs");
// Output both possible DatFiles
Writer.Write(lessThan, OutputDir);
Writer.Write(greaterThan, OutputDir);
watch.Stop();
}
// Total Size splitting
if (splittingMode.HasFlag(SplittingMode.TotalSize))
{
logger.Warning("This feature is not implemented: level-split");
List <DatFile> sizedDats = DatTools.Splitter.SplitByTotalSize(internalDat, GetInt64(features, ChunkSizeInt64Value));
InternalStopwatch watch = new InternalStopwatch("Outputting total-size-split DATs");
// Loop through each type DatFile
Parallel.ForEach(sizedDats, Globals.ParallelOptions, sizedDat =>
{
Writer.Write(sizedDat, OutputDir);
});
watch.Stop();
}
// Type splitting
if (splittingMode.HasFlag(SplittingMode.Type))
{
Dictionary <ItemType, DatFile> typeDats = DatTools.Splitter.SplitByType(internalDat);
InternalStopwatch watch = new InternalStopwatch("Outputting ItemType DATs");
// Loop through each type DatFile
Parallel.ForEach(typeDats.Keys, Globals.ParallelOptions, itemType =>
{
Writer.Write(typeDats[itemType], OutputDir);
});
watch.Stop();
}
}
return(true);
}
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
logger.Error("This feature is not yet implemented: rescan-depots");
foreach (string depotname in Inputs)
{
// Check that it's a valid depot first
if (!_depots.ContainsKey(depotname))
{
logger.User($"'{depotname}' is not a recognized depot. Please add it to your configuration file and try again");
return(false);
}
// Then check that the depot is online
if (!Directory.Exists(depotname))
{
logger.User($"'{depotname}' does not appear to be online. Please check its status and try again");
return(false);
}
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// If we have it, then check for all hashes that are in that depot
List <string> hashes = new List <string>();
string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
hashes.Add(sldr.GetString(0));
}
}
// Now rescan the depot itself
DatFile depot = DatFile.Create();
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Set the base queries to use
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Once we have both, check for any new files
List <string> dupehashes = new List <string>();
IEnumerable <string> keys = depot.Items.Keys;
foreach (string key in keys)
{
ConcurrentList <DatItem> roms = depot.Items[key];
foreach (Rom rom in roms)
{
if (hashes.Contains(rom.SHA1))
{
dupehashes.Add(rom.SHA1);
hashes.Remove(rom.SHA1);
}
else if (!dupehashes.Contains(rom.SHA1))
{
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += $" (\"{rom.CRC}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5query += $" (\"{rom.MD5}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{depotname}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
}
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
// Now that we've added the information, we get to remove all of the hashes that we want to
query = @"DELETE FROM sha1
JOIN crcsha1
ON sha1.sha1=crcsha1.sha1
JOIN md5sha1
ON sha1.sha1=md5sha1.sha1
JOIN crc
ON crcsha1.crc=crc.crc
JOIN md5
ON md5sha1.md5=md5.md5
WHERE sha1.sha1 IN ";
query += $"({string.Join("\",\"", hashes)}\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
// Dispose of the database connection
slc.Dispose();
dbc.Dispose();
}
return(true);
}
public override void ProcessFeatures(Dictionary <string, Library.Help.Feature> features)
{
base.ProcessFeatures(features);
SplittingMode splittingMode = GetSplittingMode(features);
// If we somehow have the "none" split type, return
if (splittingMode == SplittingMode.None)
{
return;
}
// Get only files from the inputs
List <ParentablePath> files = DirectoryExtensions.GetFilesOnly(Inputs, appendparent: true);
// Loop over the input files
foreach (ParentablePath file in files)
{
// Create and fill the new DAT
DatFile internalDat = DatFile.Create(Header);
internalDat.Parse(file);
// Get the output directory
OutputDir = file.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
// Extension splitting
if (splittingMode.HasFlag(SplittingMode.Extension))
{
(DatFile extADat, DatFile extBDat) = internalDat.SplitByExtension(GetList(features, ExtAListValue), GetList(features, ExtBListValue));
InternalStopwatch watch = new InternalStopwatch("Outputting extension-split DATs");
// Output both possible DatFiles
extADat.Write(OutputDir);
extBDat.Write(OutputDir);
watch.Stop();
}
// Hash splitting
if (splittingMode.HasFlag(SplittingMode.Hash))
{
Dictionary <Field, DatFile> typeDats = internalDat.SplitByHash();
InternalStopwatch watch = new InternalStopwatch("Outputting hash-split DATs");
// Loop through each type DatFile
Parallel.ForEach(typeDats.Keys, Globals.ParallelOptions, itemType =>
{
typeDats[itemType].Write(OutputDir);
});
watch.Stop();
}
// Level splitting
if (splittingMode.HasFlag(SplittingMode.Level))
{
logger.Warning("This feature is not implemented: level-split");
internalDat.SplitByLevel(
OutputDir,
GetBoolean(features, ShortValue),
GetBoolean(features, BaseValue));
}
// Size splitting
if (splittingMode.HasFlag(SplittingMode.Size))
{
(DatFile lessThan, DatFile greaterThan) = internalDat.SplitBySize(GetInt64(features, RadixInt64Value));
InternalStopwatch watch = new InternalStopwatch("Outputting size-split DATs");
// Output both possible DatFiles
lessThan.Write(OutputDir);
greaterThan.Write(OutputDir);
watch.Stop();
}
// Type splitting
if (splittingMode.HasFlag(SplittingMode.Type))
{
Dictionary <ItemType, DatFile> typeDats = internalDat.SplitByType();
InternalStopwatch watch = new InternalStopwatch("Outputting ItemType DATs");
// Loop through each type DatFile
Parallel.ForEach(typeDats.Keys, Globals.ParallelOptions, itemType =>
{
typeDats[itemType].Write(OutputDir);
});
watch.Stop();
}
}
}
public override void ProcessFeatures(Dictionary <string, Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
TreatAsFile asFiles = GetTreatAsFiles(features);
bool date = GetBoolean(features, AddDateValue);
bool delete = GetBoolean(features, DeleteValue);
bool inverse = GetBoolean(features, InverseValue);
bool quickScan = GetBoolean(features, QuickValue);
bool updateDat = GetBoolean(features, UpdateDatValue);
var outputFormat = GetOutputFormat(features);
// If we have the romba flag
if (Header.OutputDepot?.IsActive == true)
{
// Update TorrentGzip output
if (outputFormat == OutputFormat.TorrentGzip)
{
outputFormat = OutputFormat.TorrentGzipRomba;
}
// Update TorrentXz output
else if (outputFormat == OutputFormat.TorrentXZ)
{
outputFormat = OutputFormat.TorrentXZRomba;
}
}
// Get a list of files from the input datfiles
var datfiles = GetList(features, DatListValue);
var datfilePaths = DirectoryExtensions.GetFilesOnly(datfiles);
// If we are in individual mode, process each DAT on their own, appending the DAT name to the output dir
if (GetBoolean(features, IndividualValue))
{
foreach (ParentablePath datfile in datfilePaths)
{
DatFile datdata = DatFile.Create();
datdata.Parse(datfile, int.MaxValue, keep: true);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
datdata.Header.OutputDepot = Header.OutputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
// If we have the depot flag, respect it
bool success;
if (Header.InputDepot?.IsActive ?? false)
{
success = datdata.RebuildDepot(Inputs, Path.Combine(OutputDir, datdata.Header.FileName), date, delete, inverse, outputFormat);
}
else
{
success = datdata.RebuildGeneric(Inputs, Path.Combine(OutputDir, datdata.Header.FileName), quickScan, date, delete, inverse, outputFormat, asFiles);
}
// If we have a success and we're updating the DAT, write it out
if (success && updateDat)
{
datdata.Header.FileName = $"fixDAT_{Header.FileName}";
datdata.Header.Name = $"fixDAT_{Header.Name}";
datdata.Header.Description = $"fixDAT_{Header.Description}";
datdata.Items.ClearMarked();
datdata.Write(OutputDir);
}
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = DatFile.Create();
foreach (ParentablePath datfile in datfilePaths)
{
datdata.Parse(datfile, int.MaxValue, keep: true);
}
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
datdata.Header.OutputDepot = Header.OutputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
watch.Stop();
// If we have the depot flag, respect it
bool success;
if (Header.InputDepot?.IsActive ?? false)
{
success = datdata.RebuildDepot(Inputs, OutputDir, date, delete, inverse, outputFormat);
}
else
{
success = datdata.RebuildGeneric(Inputs, OutputDir, quickScan, date, delete, inverse, outputFormat, asFiles);
}
// If we have a success and we're updating the DAT, write it out
if (success && updateDat)
{
datdata.Header.FileName = $"fixDAT_{Header.FileName}";
datdata.Header.Name = $"fixDAT_{Header.Name}";
datdata.Header.Description = $"fixDAT_{Header.Description}";
datdata.Items.ClearMarked();
datdata.Write(OutputDir);
}
}
}
static void Main(string[] args)
{
string[] selection = { "Створити", "Відкрити", "Редагувати" };
string[] doctype = { "txt", "dat" };
while (true)
{
Console.WriteLine(new string('-', 30));
Console.Write("Введіть шлях до документу: ");
string path = Console.ReadLine();
Console.WriteLine(new string('-', 30));
//Файл формату .txt або .dat
AbstractHandler file;
Console.WriteLine("Виберіть тип документу");
for (int i = 0; i < doctype.Length; i++)
{
Console.WriteLine($"{i + 1}.{doctype[i]}");
}
Console.Write("---> ");
int n = Convert.ToInt32(Console.ReadLine());
switch (n)
{
case 1:
file = new TxtFile(path);
break;
case 2:
file = new DatFile(path);
break;
default:
file = null;
break;
}
Console.WriteLine(new string('-', 30));
Console.WriteLine("Виберіть, що зробити з документом");
for (int i = 0; i < selection.Length; i++)
{
Console.WriteLine($"{i + 1}.{selection[i]}");
}
Console.Write("---> ");
int m = Convert.ToInt32(Console.ReadLine());
switch (m)
{
case 1:
file.Create();
break;
case 2:
file.Open();
break;
case 3:
file.Edit();
break;
}
Console.WriteLine(new string('-', 30));
Console.WriteLine("Для продовження натисніть ENTER, щоб увійти введіть Q");
Console.Write("--->");
string exit = Console.ReadLine();
if (exit == "Q" || exit == "q")
{
break;
}
}
}
/// <summary>
/// Split a DAT by best available hashes
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <returns>Dictionary of Field to DatFile mappings</returns>
public static Dictionary <DatItemField, DatFile> SplitByHash(DatFile datFile)
{
// Create each of the respective output DATs
InternalStopwatch watch = new InternalStopwatch($"Splitting DAT by best available hashes");
// Create the set of field-to-dat mappings
Dictionary <DatItemField, DatFile> fieldDats = new Dictionary <DatItemField, DatFile>();
// TODO: Can this be made into a loop?
fieldDats[DatItemField.Status] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.Status].Header.FileName += " (Nodump)";
fieldDats[DatItemField.Status].Header.Name += " (Nodump)";
fieldDats[DatItemField.Status].Header.Description += " (Nodump)";
fieldDats[DatItemField.SHA512] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.SHA512].Header.FileName += " (SHA-512)";
fieldDats[DatItemField.SHA512].Header.Name += " (SHA-512)";
fieldDats[DatItemField.SHA512].Header.Description += " (SHA-512)";
fieldDats[DatItemField.SHA384] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.SHA384].Header.FileName += " (SHA-384)";
fieldDats[DatItemField.SHA384].Header.Name += " (SHA-384)";
fieldDats[DatItemField.SHA384].Header.Description += " (SHA-384)";
fieldDats[DatItemField.SHA256] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.SHA256].Header.FileName += " (SHA-256)";
fieldDats[DatItemField.SHA256].Header.Name += " (SHA-256)";
fieldDats[DatItemField.SHA256].Header.Description += " (SHA-256)";
fieldDats[DatItemField.SHA1] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.SHA1].Header.FileName += " (SHA-1)";
fieldDats[DatItemField.SHA1].Header.Name += " (SHA-1)";
fieldDats[DatItemField.SHA1].Header.Description += " (SHA-1)";
fieldDats[DatItemField.MD5] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.MD5].Header.FileName += " (MD5)";
fieldDats[DatItemField.MD5].Header.Name += " (MD5)";
fieldDats[DatItemField.MD5].Header.Description += " (MD5)";
fieldDats[DatItemField.CRC] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.CRC].Header.FileName += " (CRC)";
fieldDats[DatItemField.CRC].Header.Name += " (CRC)";
fieldDats[DatItemField.CRC].Header.Description += " (CRC)";
fieldDats[DatItemField.NULL] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[DatItemField.NULL].Header.FileName += " (Other)";
fieldDats[DatItemField.NULL].Header.Name += " (Other)";
fieldDats[DatItemField.NULL].Header.Description += " (Other)";
// Now populate each of the DAT objects in turn
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = datFile.Items[key];
foreach (DatItem item in items)
{
// If the file is not a Disk, Media, or Rom, continue
if (item.ItemType != ItemType.Disk && item.ItemType != ItemType.Media && item.ItemType != ItemType.Rom)
{
return;
}
// If the file is a nodump
if ((item.ItemType == ItemType.Rom && (item as Rom).ItemStatus == ItemStatus.Nodump) ||
(item.ItemType == ItemType.Disk && (item as Disk).ItemStatus == ItemStatus.Nodump))
{
fieldDats[DatItemField.Status].Items.Add(key, item);
}
// If the file has a SHA-512
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA512)))
{
fieldDats[DatItemField.SHA512].Items.Add(key, item);
}
// If the file has a SHA-384
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA384)))
{
fieldDats[DatItemField.SHA384].Items.Add(key, item);
}
// If the file has a SHA-256
else if ((item.ItemType == ItemType.Media && !string.IsNullOrWhiteSpace((item as Media).SHA256)) ||
(item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA256)))
{
fieldDats[DatItemField.SHA256].Items.Add(key, item);
}
// If the file has a SHA-1
else if ((item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace((item as Disk).SHA1)) ||
(item.ItemType == ItemType.Media && !string.IsNullOrWhiteSpace((item as Media).SHA1)) ||
(item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).SHA1)))
{
fieldDats[DatItemField.SHA1].Items.Add(key, item);
}
// If the file has an MD5
else if ((item.ItemType == ItemType.Disk && !string.IsNullOrWhiteSpace((item as Disk).MD5)) ||
(item.ItemType == ItemType.Media && !string.IsNullOrWhiteSpace((item as Media).MD5)) ||
(item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).MD5)))
{
fieldDats[DatItemField.MD5].Items.Add(key, item);
}
// If the file has a CRC
else if ((item.ItemType == ItemType.Rom && !string.IsNullOrWhiteSpace((item as Rom).CRC)))
{
fieldDats[DatItemField.CRC].Items.Add(key, item);
}
else
{
fieldDats[DatItemField.NULL].Items.Add(key, item);
}
}
});
watch.Stop();
return(fieldDats);
}
public override void ProcessFeatures(Dictionary <string, Library.Help.Feature> features)
{
base.ProcessFeatures(features);
// Try to read each input as a batch run file
foreach (string path in Inputs)
{
// If the file doesn't exist, warn but continue
if (!File.Exists(path))
{
logger.User($"{path} does not exist. Skipping...");
continue;
}
// Try to process the file now
try
{
// Every line is its own command
string[] lines = File.ReadAllLines(path);
// Each batch file has its own state
int index = 0;
DatFile datFile = DatFile.Create();
string outputDirectory = null;
// Process each command line
foreach (string line in lines)
{
// Skip empty lines
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
// Skip lines that start with REM or #
if (line.StartsWith("REM") || line.StartsWith("#"))
{
continue;
}
// Read the command in, if possible
var command = BatchCommand.Create(line);
if (command == null)
{
logger.User($"Could not process {path} due to the following line: {line}");
break;
}
// Now switch on the command
logger.User($"Attempting to invoke {command.Name} with {(command.Arguments.Count == 0 ? "no arguments" : "the following argument(s): " + string.Join(", ", command.Arguments))}");
switch (command.Name.ToLowerInvariant())
{
// Set a header field
case "set":
if (command.Arguments.Count != 2)
{
logger.User($"Invoked {command.Name} but no arguments were provided");
logger.User("Usage: set(header.field, value);");
continue;
}
// Read in the individual arguments
Field field = command.Arguments[0].AsField();
string value = command.Arguments[1];
// If we had an invalid input, log and continue
if (field == Field.NULL)
{
logger.User($"{command.Arguments[0]} was an invalid field name");
continue;
}
// Set the header field
datFile.Header.SetFields(new Dictionary <Field, string> {
[field] = value
});
break;
// Parse in new input file(s)
case "input":
if (command.Arguments.Count == 0)
{
logger.User($"Invoked {command.Name} but no arguments were provided");
logger.User("Usage: input(datpath, ...);");
continue;
}
// Get only files from inputs
List <ParentablePath> datFilePaths = DirectoryExtensions.GetFilesOnly(command.Arguments);
// Assume there could be multiple
foreach (ParentablePath datFilePath in datFilePaths)
{
datFile.Parse(datFilePath, index++);
}
break;
// Run DFD/D2D on path(s)
case "d2d":
case "dfd":
if (command.Arguments.Count == 0)
{
logger.User($"Invoked {command.Name} but no arguments were provided");
logger.User("Usage: d2d(path, ...);");
continue;
}
// TODO: Should any of the other options be added for D2D?
// Assume there could be multiple
foreach (string input in command.Arguments)
{
datFile.PopulateFromDir(input);
}
// TODO: We might not want to remove higher order hashes in the future
// TODO: We might not want to remove dates in the future
Cleaner dfdCleaner = new Cleaner()
{
ExcludeFields = Hash.DeepHashes.AsFields()
};
dfdCleaner.ExcludeFields.Add(Field.DatItem_Date);
datFile.ApplyCleaning(dfdCleaner);
break;
// Apply a filter
case "filter":
if (command.Arguments.Count < 2 || command.Arguments.Count > 4)
{
logger.User($"Invoked {command.Name} and expected between 2-4 arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: filter(field, value, [remove = false, [perMachine = false]]);");
continue;
}
// Read in the individual arguments
Field filterField = command.Arguments[0].AsField();
string filterValue = command.Arguments[1];
bool? filterRemove = false;
if (command.Arguments.Count >= 3)
{
filterRemove = command.Arguments[2].AsYesNo();
}
bool?filterPerMachine = false;
if (command.Arguments.Count >= 4)
{
filterPerMachine = command.Arguments[3].AsYesNo();
}
// If we had an invalid input, log and continue
if (filterField == Field.NULL)
{
logger.User($"{command.Arguments[0]} was an invalid field name");
continue;
}
if (filterRemove == null)
{
logger.User($"{command.Arguments[2]} was an invalid true/false value");
continue;
}
if (filterPerMachine == null)
{
logger.User($"{command.Arguments[3]} was an invalid true/false value");
continue;
}
// Create a filter with this new set of fields
Filter filter = new Filter();
filter.SetFilter(filterField, filterValue, filterRemove.Value);
// Apply the filter blindly
datFile.ApplyFilter(filter, filterPerMachine.Value);
// Cleanup after the filter
// TODO: We might not want to remove immediately
datFile.Items.ClearMarked();
datFile.Items.ClearEmpty();
break;
// Apply an extra INI
case "extra":
if (command.Arguments.Count != 2)
{
logger.User($"Invoked {command.Name} and expected 2 arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: extra(field, inipath);");
continue;
}
// Read in the individual arguments
Field extraField = command.Arguments[0].AsField();
string extraFile = command.Arguments[1];
// If we had an invalid input, log and continue
if (extraField == Field.NULL)
{
logger.User($"{command.Arguments[0]} was an invalid field name");
continue;
}
if (!File.Exists(command.Arguments[1]))
{
logger.User($"{command.Arguments[1]} was an invalid file name");
continue;
}
// Create the extra INI
ExtraIni extraIni = new ExtraIni();
ExtraIniItem extraIniItem = new ExtraIniItem();
extraIniItem.PopulateFromFile(extraFile);
extraIniItem.Field = extraField;
extraIni.Items.Add(extraIniItem);
// Apply the extra INI blindly
datFile.ApplyExtras(extraIni);
break;
// Apply internal split/merge
case "merge":
if (command.Arguments.Count != 1)
{
logger.User($"Invoked {command.Name} and expected 1 argument, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: merge(split|merged|nonmerged|full|device);");
continue;
}
// Read in the individual arguments
MergingFlag mergingFlag = command.Arguments[0].AsMergingFlag();
// If we had an invalid input, log and continue
if (mergingFlag == MergingFlag.None)
{
logger.User($"{command.Arguments[0]} was an invalid merging flag");
continue;
}
// Apply the merging flag
datFile.ApplySplitting(mergingFlag, false);
break;
// Apply description-as-name logic
case "descname":
if (command.Arguments.Count != 0)
{
logger.User($"Invoked {command.Name} and expected no arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: descname();");
continue;
}
// Apply the logic
datFile.MachineDescriptionToName();
break;
// Apply 1G1R
case "1g1r":
if (command.Arguments.Count == 0)
{
logger.User($"Invoked {command.Name} but no arguments were provided");
logger.User("Usage: 1g1r(region, ...);");
continue;
}
// Run the 1G1R functionality
datFile.OneGamePerRegion(command.Arguments);
break;
// Apply one rom per game (ORPG)
case "orpg":
if (command.Arguments.Count != 0)
{
logger.User($"Invoked {command.Name} and expected no arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: orpg();");
continue;
}
// Apply the logic
datFile.OneRomPerGame();
break;
// Remove a field
case "remove":
if (command.Arguments.Count == 0)
{
logger.User($"Invoked {command.Name} but no arguments were provided");
logger.User("Usage: remove(field, ...);");
continue;
}
// Run the removal functionality
datFile.RemoveFieldsFromItems(command.Arguments.Select(s => s.AsField()).ToList());
break;
// Apply scene date stripping
case "sds":
if (command.Arguments.Count != 0)
{
logger.User($"Invoked {command.Name} and expected no arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: sds();");
continue;
}
// Apply the logic
datFile.StripSceneDatesFromItems();
break;
// Set new output format(s)
case "format":
if (command.Arguments.Count == 0)
{
logger.User($"Invoked {command.Name} but no arguments were provided");
logger.User("Usage: format(datformat, ...);");
continue;
}
// Assume there could be multiple
datFile.Header.DatFormat = 0x00;
foreach (string format in command.Arguments)
{
datFile.Header.DatFormat |= format.AsDatFormat();
}
// If we had an invalid input, log and continue
if (datFile.Header.DatFormat == 0x00)
{
logger.User($"No valid output format found");
continue;
}
break;
// Set output directory
case "output":
if (command.Arguments.Count != 1)
{
logger.User($"Invoked {command.Name} and expected exactly 1 argument, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: output(outdir);");
continue;
}
// Only set the first as the output directory
outputDirectory = command.Arguments[0];
break;
// Write out the current DatFile
case "write":
if (command.Arguments.Count > 1)
{
logger.User($"Invoked {command.Name} and expected 0-1 arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: write([overwrite = true]);");
continue;
}
// Get overwrite value, if possible
bool?overwrite = true;
if (command.Arguments.Count == 1)
{
overwrite = command.Arguments[0].AsYesNo();
}
// If we had an invalid input, log and continue
if (overwrite == null)
{
logger.User($"{command.Arguments[0]} was an invalid true/false value");
continue;
}
// Write out the dat with the current state
datFile.Write(outputDirectory, overwrite: overwrite.Value);
break;
// Reset the internal state
case "reset":
if (command.Arguments.Count != 0)
{
logger.User($"Invoked {command.Name} and expected no arguments, but {command.Arguments.Count} arguments were provided");
logger.User("Usage: reset();");
continue;
}
// Reset all state variables
index = 0;
datFile = DatFile.Create();
outputDirectory = null;
break;
default:
logger.User($"Could not find a match for '{command.Name}'. Please see the help text for more details.");
break;
}
}
}
catch (Exception ex)
{
logger.Error(ex, $"There was an exception processing {path}");
continue;
}
}
}
public override void ProcessFeatures(Dictionary <string, Library.Help.Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
var updateFields = GetUpdateFields(features);
var updateMode = GetUpdateMode(features);
// Normalize the extensions
Header.AddExtension = (string.IsNullOrWhiteSpace(Header.AddExtension) || Header.AddExtension.StartsWith(".")
? Header.AddExtension
: $".{Header.AddExtension}");
Header.ReplaceExtension = (string.IsNullOrWhiteSpace(Header.ReplaceExtension) || Header.ReplaceExtension.StartsWith(".")
? Header.ReplaceExtension
: $".{Header.ReplaceExtension}");
// If we're in a special update mode and the names aren't set, set defaults
if (updateMode != 0)
{
// Get the values that will be used
if (string.IsNullOrWhiteSpace(Header.Date))
{
Header.Date = DateTime.Now.ToString("yyyy-MM-dd");
}
if (string.IsNullOrWhiteSpace(Header.Name))
{
Header.Name = (updateMode != 0 ? "DiffDAT" : "MergeDAT")
+ (Header.Type == "SuperDAT" ? "-SuperDAT" : string.Empty)
+ (Cleaner.DedupeRoms != DedupeType.None ? "-deduped" : string.Empty);
}
if (string.IsNullOrWhiteSpace(Header.Description))
{
Header.Description = (updateMode != 0 ? "DiffDAT" : "MergeDAT")
+ (Header.Type == "SuperDAT" ? "-SuperDAT" : string.Empty)
+ (Cleaner.DedupeRoms != DedupeType.None ? " - deduped" : string.Empty);
if (!GetBoolean(features, NoAutomaticDateValue))
{
Header.Description += $" ({Header.Date})";
}
}
if (string.IsNullOrWhiteSpace(Header.Category) && updateMode != 0)
{
Header.Category = "DiffDAT";
}
if (string.IsNullOrWhiteSpace(Header.Author))
{
Header.Author = "SabreTools";
}
}
// If no update fields are set, default to Names
if (updateFields == null || updateFields.Count == 0)
{
updateFields = new List <Field>()
{
Field.DatItem_Name
}
}
;
// Ensure we only have files in the inputs
List <ParentablePath> inputPaths = DirectoryExtensions.GetFilesOnly(Inputs, appendparent: true);
List <ParentablePath> basePaths = DirectoryExtensions.GetFilesOnly(GetList(features, BaseDatListValue));
// If we're in standard update mode, run through all of the inputs
if (updateMode == UpdateMode.None)
{
// Loop through each input and update
Parallel.ForEach(inputPaths, Globals.ParallelOptions, inputPath =>
{
// Create a new base DatFile
DatFile datFile = DatFile.Create(Header);
logger.User($"Processing '{Path.GetFileName(inputPath.CurrentPath)}'");
datFile.Parse(inputPath, keep: true,
keepext: datFile.Header.DatFormat.HasFlag(DatFormat.TSV) ||
datFile.Header.DatFormat.HasFlag(DatFormat.CSV) ||
datFile.Header.DatFormat.HasFlag(DatFormat.SSV));
// Perform additional processing steps
datFile.ApplyExtras(Extras);
datFile.ApplySplitting(GetSplitType(features), false);
datFile.ApplyFilter(Filter);
datFile.ApplyCleaning(Cleaner);
// Get the correct output path
string realOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
// Try to output the file, overwriting only if it's not in the current directory
datFile.Write(realOutDir, overwrite: GetBoolean(features, InplaceValue));
});
return;
}
// Reverse inputs if we're in a required mode
if (updateMode.HasFlag(UpdateMode.DiffReverseCascade))
{
updateMode |= UpdateMode.DiffCascade;
inputPaths.Reverse();
}
if (updateMode.HasFlag(UpdateMode.ReverseBaseReplace))
{
updateMode |= UpdateMode.BaseReplace;
basePaths.Reverse();
}
// Create a DAT to capture inputs
DatFile userInputDat = DatFile.Create(Header);
// Populate using the correct set
List <DatHeader> datHeaders;
if (updateMode.HasFlag(UpdateMode.DiffAgainst) || updateMode.HasFlag(UpdateMode.BaseReplace))
{
datHeaders = userInputDat.PopulateUserData(basePaths);
}
else
{
datHeaders = userInputDat.PopulateUserData(inputPaths);
}
// Perform additional processing steps
userInputDat.ApplyExtras(Extras);
userInputDat.ApplySplitting(GetSplitType(features), false);
userInputDat.ApplyFilter(Filter);
userInputDat.ApplyCleaning(Cleaner);
// Output only DatItems that are duplicated across inputs
if (updateMode.HasFlag(UpdateMode.DiffDupesOnly))
{
DatFile dupeData = userInputDat.DiffDuplicates(inputPaths);
InternalStopwatch watch = new InternalStopwatch("Outputting duplicate DAT");
dupeData.Write(OutputDir, overwrite: false);
watch.Stop();
}
// Output only DatItems that are not duplicated across inputs
if (updateMode.HasFlag(UpdateMode.DiffNoDupesOnly))
{
DatFile outerDiffData = userInputDat.DiffNoDuplicates(inputPaths);
InternalStopwatch watch = new InternalStopwatch("Outputting no duplicate DAT");
outerDiffData.Write(OutputDir, overwrite: false);
watch.Stop();
}
// Output only DatItems that are unique to each input
if (updateMode.HasFlag(UpdateMode.DiffIndividualsOnly))
{
// Get all of the output DatFiles
List <DatFile> datFiles = userInputDat.DiffIndividuals(inputPaths);
// Loop through and output the new DatFiles
InternalStopwatch watch = new InternalStopwatch("Outputting all individual DATs");
Parallel.For(0, inputPaths.Count, Globals.ParallelOptions, j =>
{
string path = inputPaths[j].GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
// Try to output the file
datFiles[j].Write(path, overwrite: GetBoolean(features, InplaceValue));
});
watch.Stop();
}
// Output cascaded diffs
if (updateMode.HasFlag(UpdateMode.DiffCascade))
{
// Preprocess the DatHeaders
Parallel.For(0, datHeaders.Count, Globals.ParallelOptions, j =>
{
// If we're outputting to the runtime folder, rename
if (!GetBoolean(features, InplaceValue) && OutputDir == Environment.CurrentDirectory)
{
string innerpost = $" ({j} - {inputPaths[j].GetNormalizedFileName(true)} Only)";
datHeaders[j] = userInputDat.Header;
datHeaders[j].FileName += innerpost;
datHeaders[j].Name += innerpost;
datHeaders[j].Description += innerpost;
}
});
// Get all of the output DatFiles
List <DatFile> datFiles = userInputDat.DiffCascade(datHeaders);
// Loop through and output the new DatFiles
InternalStopwatch watch = new InternalStopwatch("Outputting all created DATs");
int startIndex = GetBoolean(features, SkipFirstOutputValue) ? 1 : 0;
Parallel.For(startIndex, inputPaths.Count, Globals.ParallelOptions, j =>
{
string path = inputPaths[j].GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
// Try to output the file
datFiles[j].Write(path, overwrite: GetBoolean(features, InplaceValue));
});
watch.Stop();
}
// Output differences against a base DAT
if (updateMode.HasFlag(UpdateMode.DiffAgainst))
{
// Loop through each input and diff against the base
Parallel.ForEach(inputPaths, Globals.ParallelOptions, inputPath =>
{
// Parse the path to a new DatFile
DatFile repDat = DatFile.Create(userInputDat.Header.CloneFiltering());
repDat.Parse(inputPath, indexId: 1, keep: true);
// Perform additional processing steps
repDat.ApplyExtras(Extras);
repDat.ApplySplitting(GetSplitType(features), false);
repDat.ApplyFilter(Filter);
repDat.ApplyCleaning(Cleaner);
// Now replace the fields from the base DatFile
userInputDat.DiffAgainst(repDat, GetBoolean(Features, ByGameValue));
// Finally output the diffed DatFile
string interOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
repDat.Write(interOutDir, overwrite: GetBoolean(features, InplaceValue));
});
}
// Output DATs after replacing fields from a base DatFile
if (updateMode.HasFlag(UpdateMode.BaseReplace))
{
// Loop through each input and apply the base DatFile
Parallel.ForEach(inputPaths, Globals.ParallelOptions, inputPath =>
{
// Parse the path to a new DatFile
DatFile repDat = DatFile.Create(userInputDat.Header.CloneFiltering());
repDat.Parse(inputPath, indexId: 1, keep: true);
// Perform additional processing steps
repDat.ApplyExtras(Extras);
repDat.ApplySplitting(GetSplitType(features), false);
repDat.ApplyFilter(Filter);
repDat.ApplyCleaning(Cleaner);
// Now replace the fields from the base DatFile
userInputDat.BaseReplace(repDat, updateFields, GetBoolean(features, OnlySameValue));
// Finally output the replaced DatFile
string interOutDir = inputPath.GetOutputPath(OutputDir, GetBoolean(features, InplaceValue));
repDat.Write(interOutDir, overwrite: GetBoolean(features, InplaceValue));
});
}
// Merge all input files and write
// This has to be last due to the SuperDAT handling
if (updateMode.HasFlag(UpdateMode.Merge))
{
// If we're in SuperDAT mode, prefix all games with their respective DATs
if (string.Equals(userInputDat.Header.Type, "SuperDAT", StringComparison.OrdinalIgnoreCase))
{
userInputDat.ApplySuperDAT(inputPaths);
}
userInputDat.Write(OutputDir);
}
}
}
public override void ProcessFeatures(Dictionary <string, Feature> features)
{
base.ProcessFeatures(features);
// Get a list of files from the input datfiles
var datfiles = GetList(features, DatListValue);
var datfilePaths = DirectoryExtensions.GetFilesOnly(datfiles);
// Get feature flags
TreatAsFile asFiles = GetTreatAsFiles(features);
bool hashOnly = GetBoolean(features, HashOnlyValue);
bool quickScan = GetBoolean(features, QuickValue);
var splitType = GetSplitType(features);
// If we are in individual mode, process each DAT on their own
if (GetBoolean(features, IndividualValue))
{
foreach (ParentablePath datfile in datfilePaths)
{
// Parse in from the file
DatFile datdata = DatFile.Create();
datdata.Parse(datfile, int.MaxValue, keep: true);
// Perform additional processing steps
datdata.ApplyExtras(Extras);
datdata.ApplySplitting(splitType, true);
datdata.ApplyFilter(Filter);
datdata.ApplyCleaning(Cleaner);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
// If we have the depot flag, respect it
if (Header.InputDepot?.IsActive ?? false)
{
datdata.VerifyDepot(Inputs);
}
else
{
// Loop through and add the inputs to check against
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
datdata.PopulateFromDir(input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
}
datdata.VerifyGeneric(hashOnly);
}
// Now write out if there are any items left
datdata.WriteStatsToConsole();
datdata.Write(OutputDir);
}
}
// Otherwise, process all DATs into the same output
else
{
InternalStopwatch watch = new InternalStopwatch("Populating internal DAT");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = DatFile.Create();
foreach (ParentablePath datfile in datfilePaths)
{
datdata.Parse(datfile, int.MaxValue, keep: true);
}
// Perform additional processing steps
datdata.ApplyExtras(Extras);
datdata.ApplySplitting(splitType, true);
datdata.ApplyFilter(Filter);
datdata.ApplyCleaning(Cleaner);
// Set depot information
datdata.Header.InputDepot = Header.InputDepot.Clone() as DepotInformation;
// If we have overridden the header skipper, set it now
if (!string.IsNullOrEmpty(Header.HeaderSkipper))
{
datdata.Header.HeaderSkipper = Header.HeaderSkipper;
}
watch.Stop();
// If we have the depot flag, respect it
if (Header.InputDepot?.IsActive ?? false)
{
datdata.VerifyDepot(Inputs);
}
else
{
// Loop through and add the inputs to check against
logger.User("Processing files:\n");
foreach (string input in Inputs)
{
datdata.PopulateFromDir(input, asFiles: asFiles, hashes: quickScan ? Hash.CRC : Hash.Standard);
}
datdata.VerifyGeneric(hashOnly);
}
// Now write out if there are any items left
datdata.WriteStatsToConsole();
datdata.Write(OutputDir);
}
}
/// <summary>
/// Output non-cascading diffs
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="inputs">List of inputs to write out from</param>
public static List <DatFile> DiffIndividuals(DatFile datFile, List <ParentablePath> inputs)
{
InternalStopwatch watch = new InternalStopwatch("Initializing all individual DATs");
// Fill in any information not in the base DAT
if (string.IsNullOrWhiteSpace(datFile.Header.FileName))
{
datFile.Header.FileName = "All DATs";
}
if (string.IsNullOrWhiteSpace(datFile.Header.Name))
{
datFile.Header.Name = "All DATs";
}
if (string.IsNullOrWhiteSpace(datFile.Header.Description))
{
datFile.Header.Description = "All DATs";
}
// Loop through each of the inputs and get or create a new DatData object
DatFile[] outDatsArray = new DatFile[inputs.Count];
Parallel.For(0, inputs.Count, Globals.ParallelOptions, j =>
{
string innerpost = $" ({j} - {inputs[j].GetNormalizedFileName(true)} Only)";
DatFile diffData = DatFile.Create(datFile.Header);
diffData.Header.FileName += innerpost;
diffData.Header.Name += innerpost;
diffData.Header.Description += innerpost;
diffData.Items = new ItemDictionary();
outDatsArray[j] = diffData;
});
// Create a list of DatData objects representing individual output files
List <DatFile> outDats = outDatsArray.ToList();
watch.Stop();
// Now, loop through the dictionary and populate the correct DATs
watch.Start("Populating all individual DATs");
Parallel.ForEach(datFile.Items.Keys, Globals.ParallelOptions, key =>
{
ConcurrentList <DatItem> items = DatItem.Merge(datFile.Items[key]);
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
{
return;
}
// Loop through and add the items correctly
foreach (DatItem item in items)
{
if (item.DupeType.HasFlag(DupeType.Internal) || item.DupeType == 0x00)
{
outDats[item.Source.Index].Items.Add(key, item);
}
}
});
watch.Stop();
return(outDats.ToList());
}
public override void ProcessFeatures(Dictionary <string, Library.Help.Feature> features)
{
base.ProcessFeatures(features);
// Get feature flags
bool addBlankFiles = GetBoolean(features, AddBlankFilesValue);
bool addFileDates = GetBoolean(features, AddDateValue);
TreatAsFile asFiles = GetTreatAsFiles(features);
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
var splitType = GetSplitType(features);
// Apply the specialized field removals to the cleaner
if (Cleaner.ExcludeFields == null)
{
Cleaner.ExcludeFields = new List <Field>();
}
if (!addFileDates)
{
Cleaner.ExcludeFields.Add(Field.DatItem_Date);
}
// Create a new DATFromDir object and process the inputs
DatFile basedat = DatFile.Create(Header);
basedat.Header.Date = DateTime.Now.ToString("yyyy-MM-dd");
// For each input directory, create a DAT
foreach (string path in Inputs)
{
if (Directory.Exists(path) || File.Exists(path))
{
// Clone the base Dat for information
DatFile datdata = DatFile.Create(basedat.Header);
// Get the base path and fill the header, if needed
string basePath = Path.GetFullPath(path);
datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path
bool success = datdata.PopulateFromDir(
basePath,
asFiles,
skipFileType,
addBlankFiles,
hashes: includeInScan);
if (success)
{
// Perform additional processing steps
datdata.ApplyExtras(Extras);
datdata.ApplySplitting(splitType, false);
datdata.ApplyFilter(Filter);
datdata.ApplyCleaning(Cleaner);
// Write out the file
datdata.Write(OutputDir);
}
else
{
Console.WriteLine();
OutputRecursive(0);
}
}
}
}
public override bool ProcessFeatures(Dictionary <string, Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
bool addBlankFiles = GetBoolean(features, AddBlankFilesValue);
bool addFileDates = GetBoolean(features, AddDateValue);
TreatAsFile asFiles = GetTreatAsFiles(features);
bool noAutomaticDate = GetBoolean(features, NoAutomaticDateValue);
var includeInScan = GetIncludeInScan(features);
var skipFileType = GetSkipFileType(features);
// Apply the specialized field removals to the cleaner
if (!addFileDates)
{
Remover.PopulateExclusionsFromList(new List <string> {
"DatItem.Date"
});
}
// Create a new DATFromDir object and process the inputs
DatFile basedat = DatFile.Create(Header);
basedat.Header.Date = DateTime.Now.ToString("yyyy-MM-dd");
// For each input directory, create a DAT
foreach (string path in Inputs)
{
if (Directory.Exists(path) || File.Exists(path))
{
// Clone the base Dat for information
DatFile datdata = DatFile.Create(basedat.Header);
// Get the base path and fill the header, if needed
string basePath = Path.GetFullPath(path);
datdata.FillHeaderFromPath(basePath, noAutomaticDate);
// Now populate from the path
bool success = DatTools.DatFromDir.PopulateFromDir(
datdata,
basePath,
asFiles,
skipFileType,
addBlankFiles,
hashes: includeInScan);
if (success)
{
// Perform additional processing steps
Extras.ApplyExtras(datdata);
Splitter.ApplySplitting(datdata, false);
Filter.ApplyFilters(datdata);
Cleaner.ApplyCleaning(datdata);
Remover.ApplyRemovals(datdata);
// Write out the file
Writer.Write(datdata, OutputDir);
}
else
{
Console.WriteLine();
OutputRecursive(0);
}
}
}
return(true);
}
/// <summary>
/// Split a DAT by size of Rom
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="chunkSize">Long value representing the total size to split at</param>
/// <returns>Less Than and Greater Than DatFiles</returns>
public static List <DatFile> SplitByTotalSize(DatFile datFile, long chunkSize)
{
// If the size is invalid, just return
if (chunkSize <= 0)
{
return(new List <DatFile>());
}
// Create each of the respective output DATs
InternalStopwatch watch = new InternalStopwatch($"Splitting DAT by total size");
// Sort the DatFile by machine name
datFile.Items.BucketBy(ItemKey.Machine, DedupeType.None);
// Get the keys in a known order for easier sorting
var keys = datFile.Items.SortedKeys;
// Get the output list
List <DatFile> datFiles = new List <DatFile>();
// Initialize everything
long currentSize = 0;
long currentIndex = 0;
DatFile currentDat = DatFile.Create(datFile.Header.CloneStandard());
currentDat.Header.FileName += $"_{currentIndex}";
currentDat.Header.Name += $"_{currentIndex}";
currentDat.Header.Description += $"_{currentIndex}";
// Loop through each machine
foreach (string machine in keys)
{
// Get the current machine
var items = datFile.Items[machine];
if (items == null || !items.Any())
{
logger.Error($"{machine} contains no items and will be skipped");
continue;
}
// Get the total size of the current machine
long machineSize = 0;
foreach (var item in items)
{
if (item is Rom rom)
{
// TODO: Should there be more than just a log if a single item is larger than the chunksize?
machineSize += rom.Size ?? 0;
if ((rom.Size ?? 0) > chunkSize)
{
logger.Error($"{rom.GetName() ?? string.Empty} in {machine} is larger than {chunkSize}");
}
}
}
// If the current machine size is greater than the chunk size by itself, we want to log and skip
// TODO: Should this eventually try to split the machine here?
if (machineSize > chunkSize)
{
logger.Error($"{machine} is larger than {chunkSize} and will be skipped");
continue;
}
// If the current machine size makes the current DatFile too big, split
else if (currentSize + machineSize > chunkSize)
{
datFiles.Add(currentDat);
currentSize = 0;
currentIndex++;
currentDat = DatFile.Create(datFile.Header.CloneStandard());
currentDat.Header.FileName += $"_{currentIndex}";
currentDat.Header.Name += $"_{currentIndex}";
currentDat.Header.Description += $"_{currentIndex}";
}
// Add the current machine to the current DatFile
currentDat.Items[machine] = items;
currentSize += machineSize;
}
// Add the final DatFile to the list
datFiles.Add(currentDat);
// Then return the list
watch.Stop();
return(datFiles);
}
static void Main(string[] args)
{
string[] selection = { "Создать", "Открыть", "Редактировать" };
string[] doctype = { "txt", "dat" };
while (true)
{
Console.WriteLine(new string('-', 30));
Console.Write("Введите путь к документу: ");
string path = Console.ReadLine();
Console.WriteLine(new string('-', 30));
//Файл формата .txt или .dat
AbstractHandler file;
Console.WriteLine("Выберите тип документа");
for (int i = 0; i < doctype.Length; i++)
{
Console.WriteLine($"{i + 1}.{doctype[i]}");
}
Console.Write("---> ");
int n = Convert.ToInt32(Console.ReadLine());
switch (n)
{
case 1:
file = new TxtFile(path);
break;
case 2:
file = new DatFile(path);
break;
default:
file = null;
break;
}
Console.WriteLine(new string('-', 30));
Console.WriteLine("Выберите что сделать с документом");
for (int i = 0; i < selection.Length; i++)
{
Console.WriteLine($"{i + 1}.{selection[i]}");
}
Console.Write("---> ");
int m = Convert.ToInt32(Console.ReadLine());
switch (m)
{
case 1:
file.Create();
break;
case 2:
file.Open();
break;
case 3:
file.Edit();
break;
}
Console.WriteLine(new string('-', 30));
Console.WriteLine("Для продолжения нажмите ENTER, чтобы выйти введите Q");
Console.Write("--->");
string exit = Console.ReadLine();
if (exit == "Q" || exit == "q")
{
break;
}
}
}
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
int workers = GetInt32(features, WorkersInt32Value);
string missingSha1s = GetString(features, MissingSha1sStringValue);
// Make sure the db is set
if (string.IsNullOrWhiteSpace(_db))
{
_db = "db.sqlite";
_connectionString = $"Data Source={_db};Version = 3;";
}
// Make sure the file exists
if (!File.Exists(_db))
{
EnsureDatabase(_db, _connectionString);
}
// Make sure the dats dir is set
if (string.IsNullOrWhiteSpace(_dats))
{
_dats = "dats";
}
_dats = Path.Combine(PathTool.GetRuntimeDirectory(), _dats);
// Make sure the folder exists
if (!Directory.Exists(_dats))
{
Directory.CreateDirectory(_dats);
}
// First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create();
datroot.Header.Type = "SuperDAT";
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1)
List <string> databaseDats = new List <string>();
List <string> unneeded = new List <string>();
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Populate the List from the database
InternalStopwatch watch = new InternalStopwatch("Populating the list of existing DATs");
string query = "SELECT DISTINCT hash FROM dat";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
sldr.Read();
string hash = sldr.GetString(0);
if (datroot.Items.ContainsKey(hash))
{
datroot.Items.Remove(hash);
databaseDats.Add(hash);
}
else if (!databaseDats.Contains(hash))
{
unneeded.Add(hash);
}
}
datroot.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
watch.Stop();
slc.Dispose();
sldr.Dispose();
// Loop through the Dictionary and add all data
watch.Start("Adding new DAT information");
foreach (string key in datroot.Items.Keys)
{
foreach (Rom value in datroot.Items[key])
{
AddDatToDatabase(value, dbc);
}
}
watch.Stop();
// Now loop through and remove all references to old Dats
if (unneeded.Count > 0)
{
watch.Start("Removing unmatched DAT information");
query = "DELETE FROM dat WHERE";
foreach (string dathash in unneeded)
{
query += $" OR hash=\"{dathash}\"";
}
query = query.Replace("WHERE OR", "WHERE");
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
slc.Dispose();
watch.Stop();
}
dbc.Dispose();
return(true);
}
