public override bool ProcessFeatures(Dictionary <string, Feature> features) { // If the base fails, just fail out if (!base.ProcessFeatures(features)) { return(false); } // Get feature flags string name = GetString(features, NameStringValue); string description = GetString(features, DescriptionStringValue); string source = GetString(features, SourceStringValue); string outdat = GetString(features, OutStringValue); // Ensure the output directory outdat.Ensure(create: true); // Check that all required directories exist if (!Directory.Exists(source)) { logger.Error($"File '{source}' does not exist!"); return(false); } // Create and write the encapsulating datfile DatFile datfile = DatFile.Create(); datfile.Header.Name = string.IsNullOrWhiteSpace(name) ? "untitled" : name; datfile.Header.Description = description; DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard); Writer.Write(datfile, outdat); return(true); }
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features) { // If the base fails, just fail out if (!base.ProcessFeatures(features)) { return(false); } // Get the archive scanning level // TODO: Remove usage int sevenzip = GetInt32(features, Include7ZipsInt32Value); int gz = GetInt32(features, IncludeGZipsInt32Value); int zip = GetInt32(features, IncludeZipsInt32Value); // Get feature flags bool noDb = GetBoolean(features, NoDbValue); bool onlyNeeded = GetBoolean(features, OnlyNeededValue); // First we want to get just all directories from the inputs List <string> onlyDirs = new List <string>(); foreach (string input in Inputs) { if (Directory.Exists(input)) { onlyDirs.Add(Path.GetFullPath(input)); } } // Then process all of the input directories into an internal DAT DatFile df = DatFile.Create(); foreach (string dir in onlyDirs) { DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard); DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: Hash.Standard); } // Create an empty Dat for files that need to be rebuilt DatFile need = DatFile.Create(); // Open the database connection SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); // Now that we have the Dats, add the files to the database string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES"; string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; foreach (string key in df.Items.Keys) { ConcurrentList <DatItem> datItems = df.Items[key]; foreach (Rom rom in datItems) { // If we care about if the file exists, check the databse first if (onlyNeeded && !noDb) { string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1" + $" WHERE crcsha1.crc=\"{rom.CRC}\"" + $" OR md5sha1.md5=\"{rom.MD5}\"" + $" OR md5sha1.sha1=\"{rom.SHA1}\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) { // Add to the queries if (!string.IsNullOrWhiteSpace(rom.CRC)) { crcquery += $" (\"{rom.CRC}\"),"; } if (!string.IsNullOrWhiteSpace(rom.MD5)) { md5query += $" (\"{rom.MD5}\"),"; } if (!string.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),"; if (!string.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),"; } if (!string.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),"; } } // Add to the Dat need.Items.Add(key, rom); } } // Otherwise, just add the file to the list else { // Add to the queries if (!noDb) { if (!string.IsNullOrWhiteSpace(rom.CRC)) { crcquery += $" (\"{rom.CRC}\"),"; } if (!string.IsNullOrWhiteSpace(rom.MD5)) { md5query += $" (\"{rom.MD5}\"),"; } if (!string.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),"; if (!string.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),"; } if (!string.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),"; } } } // Add to the Dat need.Items.Add(key, rom); } } } // Now run the queries, if they're populated if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") { SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") { SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES") { SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") { SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") { SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } // Create the sorting object to use and rebuild the needed files Rebuilder.RebuildGeneric( need, onlyDirs, outDir: _depots.Keys.ToList()[0], outputFormat: OutputFormat.TorrentGzipRomba, asFiles: TreatAsFile.NonArchive); return(true); }
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features) { // If the base fails, just fail out if (!base.ProcessFeatures(features)) { return(false); } logger.Error("This feature is not yet implemented: rescan-depots"); foreach (string depotname in Inputs) { // Check that it's a valid depot first if (!_depots.ContainsKey(depotname)) { logger.User($"'{depotname}' is not a recognized depot. Please add it to your configuration file and try again"); return(false); } // Then check that the depot is online if (!Directory.Exists(depotname)) { logger.User($"'{depotname}' does not appear to be online. Please check its status and try again"); return(false); } // Open the database connection SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); // If we have it, then check for all hashes that are in that depot List <string> hashes = new List <string>(); string query = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) { while (sldr.Read()) { hashes.Add(sldr.GetString(0)); } } // Now rescan the depot itself DatFile depot = DatFile.Create(); DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard); depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); // Set the base queries to use string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES"; string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; // Once we have both, check for any new files List <string> dupehashes = new List <string>(); IEnumerable <string> keys = depot.Items.Keys; foreach (string key in keys) { ConcurrentList <DatItem> roms = depot.Items[key]; foreach (Rom rom in roms) { if (hashes.Contains(rom.SHA1)) { dupehashes.Add(rom.SHA1); hashes.Remove(rom.SHA1); } else if (!dupehashes.Contains(rom.SHA1)) { if (!string.IsNullOrWhiteSpace(rom.CRC)) { crcquery += $" (\"{rom.CRC}\"),"; } if (!string.IsNullOrWhiteSpace(rom.MD5)) { md5query += $" (\"{rom.MD5}\"),"; } if (!string.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += $" (\"{rom.SHA1}\", \"{depotname}\"),"; if (!string.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),"; } if (!string.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),"; } } } } } // Now run the queries after fixing them if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") { slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); slc.ExecuteNonQuery(); } if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") { slc = new SqliteCommand(md5query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); } if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES") { slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); } if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") { slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); } if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") { slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); } // Now that we've added the information, we get to remove all of the hashes that we want to query = @"DELETE FROM sha1 JOIN crcsha1 ON sha1.sha1=crcsha1.sha1 JOIN md5sha1 ON sha1.sha1=md5sha1.sha1 JOIN crc ON crcsha1.crc=crc.crc JOIN md5 ON md5sha1.md5=md5.md5 WHERE sha1.sha1 IN "; query += $"({string.Join("\",\"", hashes)}\")"; slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery(); // Dispose of the database connection slc.Dispose(); dbc.Dispose(); } return(true); }
public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features) { // If the base fails, just fail out if (!base.ProcessFeatures(features)) { return(false); } // Get feature flags int workers = GetInt32(features, WorkersInt32Value); string missingSha1s = GetString(features, MissingSha1sStringValue); // Make sure the db is set if (string.IsNullOrWhiteSpace(_db)) { _db = "db.sqlite"; _connectionString = $"Data Source={_db};Version = 3;"; } // Make sure the file exists if (!File.Exists(_db)) { EnsureDatabase(_db, _connectionString); } // Make sure the dats dir is set if (string.IsNullOrWhiteSpace(_dats)) { _dats = "dats"; } _dats = Path.Combine(PathTool.GetRuntimeDirectory(), _dats); // Make sure the folder exists if (!Directory.Exists(_dats)) { Directory.CreateDirectory(_dats); } // First get a list of SHA-1's from the input DATs DatFile datroot = DatFile.Create(); datroot.Header.Type = "SuperDAT"; DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard); datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None); // Create a List of dat hashes in the database (SHA-1) List <string> databaseDats = new List <string>(); List <string> unneeded = new List <string>(); SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); // Populate the List from the database InternalStopwatch watch = new InternalStopwatch("Populating the list of existing DATs"); string query = "SELECT DISTINCT hash FROM dat"; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) { sldr.Read(); string hash = sldr.GetString(0); if (datroot.Items.ContainsKey(hash)) { datroot.Items.Remove(hash); databaseDats.Add(hash); } else if (!databaseDats.Contains(hash)) { unneeded.Add(hash); } } datroot.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true); watch.Stop(); slc.Dispose(); sldr.Dispose(); // Loop through the Dictionary and add all data watch.Start("Adding new DAT information"); foreach (string key in datroot.Items.Keys) { foreach (Rom value in datroot.Items[key]) { AddDatToDatabase(value, dbc); } } watch.Stop(); // Now loop through and remove all references to old Dats if (unneeded.Count > 0) { watch.Start("Removing unmatched DAT information"); query = "DELETE FROM dat WHERE"; foreach (string dathash in unneeded) { query += $" OR hash=\"{dathash}\""; } query = query.Replace("WHERE OR", "WHERE"); slc = new SqliteCommand(query, dbc); slc.ExecuteNonQuery(); slc.Dispose(); watch.Stop(); } dbc.Dispose(); return(true); }