public override bool ProcessFeatures(Dictionary <string, Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
string name = GetString(features, NameStringValue);
string description = GetString(features, DescriptionStringValue);
string source = GetString(features, SourceStringValue);
string outdat = GetString(features, OutStringValue);
// Ensure the output directory
outdat.Ensure(create: true);
// Check that all required directories exist
if (!Directory.Exists(source))
{
logger.Error($"File '{source}' does not exist!");
return(false);
}
// Create and write the encapsulating datfile
DatFile datfile = DatFile.Create();
datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name;
datfile.Header.Description = description;
DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
Writer.Write(datfile, outdat);
return(true);
}
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get the archive scanning level
// TODO: Remove usage
int sevenzip = GetInt32(features, Include7ZipsInt32Value);
int gz = GetInt32(features, IncludeGZipsInt32Value);
int zip = GetInt32(features, IncludeZipsInt32Value);
// Get feature flags
bool noDb = GetBoolean(features, NoDbValue);
bool onlyNeeded = GetBoolean(features, OnlyNeededValue);
// First we want to get just all directories from the inputs
List <string> onlyDirs = new List <string>();
foreach (string input in Inputs)
{
if (Directory.Exists(input))
{
onlyDirs.Add(Path.GetFullPath(input));
}
}
// Then process all of the input directories into an internal DAT
DatFile df = DatFile.Create();
foreach (string dir in onlyDirs)
{
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: Hash.Standard);
}
// Create an empty Dat for files that need to be rebuilt
DatFile need = DatFile.Create();
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Now that we have the Dats, add the files to the database
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
foreach (string key in df.Items.Keys)
{
ConcurrentList <DatItem> datItems = df.Items[key];
foreach (Rom rom in datItems)
{
// If we care about if the file exists, check the databse first
if (onlyNeeded && !noDb)
{
string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"
+ $" WHERE crcsha1.crc=\"{rom.CRC}\""
+ $" OR md5sha1.md5=\"{rom.MD5}\""
+ $" OR md5sha1.sha1=\"{rom.SHA1}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
// Add to the queries
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += $" (\"{rom.CRC}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5query += $" (\"{rom.MD5}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
// Add to the Dat
need.Items.Add(key, rom);
}
}
// Otherwise, just add the file to the list
else
{
// Add to the queries
if (!noDb)
{
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += $" (\"{rom.CRC}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5query += $" (\"{rom.MD5}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
}
// Add to the Dat
need.Items.Add(key, rom);
}
}
}
// Now run the queries, if they're populated
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
slc.Dispose();
}
// Create the sorting object to use and rebuild the needed files
Rebuilder.RebuildGeneric(
need,
onlyDirs,
outDir: _depots.Keys.ToList()[0],
outputFormat: OutputFormat.TorrentGzipRomba,
asFiles: TreatAsFile.NonArchive);
return(true);
}
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
logger.Error("This feature is not yet implemented: rescan-depots");
foreach (string depotname in Inputs)
{
// Check that it's a valid depot first
if (!_depots.ContainsKey(depotname))
{
logger.User($"'{depotname}' is not a recognized depot. Please add it to your configuration file and try again");
return(false);
}
// Then check that the depot is online
if (!Directory.Exists(depotname))
{
logger.User($"'{depotname}' does not appear to be online. Please check its status and try again");
return(false);
}
// Open the database connection
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// If we have it, then check for all hashes that are in that depot
List <string> hashes = new List <string>();
string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\"";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
while (sldr.Read())
{
hashes.Add(sldr.GetString(0));
}
}
// Now rescan the depot itself
DatFile depot = DatFile.Create();
DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Set the base queries to use
string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES";
string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES";
string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";
// Once we have both, check for any new files
List <string> dupehashes = new List <string>();
IEnumerable <string> keys = depot.Items.Keys;
foreach (string key in keys)
{
ConcurrentList <DatItem> roms = depot.Items[key];
foreach (Rom rom in roms)
{
if (hashes.Contains(rom.SHA1))
{
dupehashes.Add(rom.SHA1);
hashes.Remove(rom.SHA1);
}
else if (!dupehashes.Contains(rom.SHA1))
{
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcquery += $" (\"{rom.CRC}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5query += $" (\"{rom.MD5}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
sha1query += $" (\"{rom.SHA1}\", \"{depotname}\"),";
if (!string.IsNullOrWhiteSpace(rom.CRC))
{
crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
}
if (!string.IsNullOrWhiteSpace(rom.MD5))
{
md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
}
}
}
}
}
// Now run the queries after fixing them
if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
{
slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
{
slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
{
slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
{
slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
{
slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
slc.ExecuteNonQuery();
}
// Now that we've added the information, we get to remove all of the hashes that we want to
query = @"DELETE FROM sha1
JOIN crcsha1
ON sha1.sha1=crcsha1.sha1
JOIN md5sha1
ON sha1.sha1=md5sha1.sha1
JOIN crc
ON crcsha1.crc=crc.crc
JOIN md5
ON md5sha1.md5=md5.md5
WHERE sha1.sha1 IN ";
query += $"({string.Join("\",\"", hashes)}\")";
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
// Dispose of the database connection
slc.Dispose();
dbc.Dispose();
}
return(true);
}
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
{
// If the base fails, just fail out
if (!base.ProcessFeatures(features))
{
return(false);
}
// Get feature flags
int workers = GetInt32(features, WorkersInt32Value);
string missingSha1s = GetString(features, MissingSha1sStringValue);
// Make sure the db is set
if (string.IsNullOrWhiteSpace(_db))
{
_db = "db.sqlite";
_connectionString = $"Data Source={_db};Version = 3;";
}
// Make sure the file exists
if (!File.Exists(_db))
{
EnsureDatabase(_db, _connectionString);
}
// Make sure the dats dir is set
if (string.IsNullOrWhiteSpace(_dats))
{
_dats = "dats";
}
_dats = Path.Combine(PathTool.GetRuntimeDirectory(), _dats);
// Make sure the folder exists
if (!Directory.Exists(_dats))
{
Directory.CreateDirectory(_dats);
}
// First get a list of SHA-1's from the input DATs
DatFile datroot = DatFile.Create();
datroot.Header.Type = "SuperDAT";
DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);
// Create a List of dat hashes in the database (SHA-1)
List <string> databaseDats = new List <string>();
List <string> unneeded = new List <string>();
SqliteConnection dbc = new SqliteConnection(_connectionString);
dbc.Open();
// Populate the List from the database
InternalStopwatch watch = new InternalStopwatch("Populating the list of existing DATs");
string query = "SELECT DISTINCT hash FROM dat";
SqliteCommand slc = new SqliteCommand(query, dbc);
SqliteDataReader sldr = slc.ExecuteReader();
if (sldr.HasRows)
{
sldr.Read();
string hash = sldr.GetString(0);
if (datroot.Items.ContainsKey(hash))
{
datroot.Items.Remove(hash);
databaseDats.Add(hash);
}
else if (!databaseDats.Contains(hash))
{
unneeded.Add(hash);
}
}
datroot.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);
watch.Stop();
slc.Dispose();
sldr.Dispose();
// Loop through the Dictionary and add all data
watch.Start("Adding new DAT information");
foreach (string key in datroot.Items.Keys)
{
foreach (Rom value in datroot.Items[key])
{
AddDatToDatabase(value, dbc);
}
}
watch.Stop();
// Now loop through and remove all references to old Dats
if (unneeded.Count > 0)
{
watch.Start("Removing unmatched DAT information");
query = "DELETE FROM dat WHERE";
foreach (string dathash in unneeded)
{
query += $" OR hash=\"{dathash}\"";
}
query = query.Replace("WHERE OR", "WHERE");
slc = new SqliteCommand(query, dbc);
slc.ExecuteNonQuery();
slc.Dispose();
watch.Stop();
}
dbc.Dispose();
return(true);
}
