/// <summary> /// Wrap adding files to the depots /// </summary> /// <param name="inputs">List of input folders to use</param> /// <param name="onlyNeeded">True if only files in the database and don't exist are added, false otherwise</param> /// <param name="resume">Resume a previously interrupted operation from the specified path</param> /// <param name="includeZips">flag value == 0 means: add Zip files themselves into the depot in addition to their contents, flag value == 2 means add Zip files themselves but don't add content</param> /// <param name="workers">How many workers to launch for the job, default from config</param> /// <param name="includeGZips">flag value == 0 means: add GZip files themselves into the depot in addition to their contents, flag value == 2 means add GZip files themselves but don't add content</param> /// <param name="include7Zips">flag value == 0 means: add 7Zip files themselves into the depot in addition to their contents, flag value == 2 means add 7Zip files themselves but don't add content</param> /// <param name="skipInitialScan">True to skip the initial scan of the files to determine amount of work, false otherwise</param> /// <param name="useGolangZip">True to use go zip implementation instead of zlib, false otherwise</param> /// <param name="noDb">True to archive into depot but do not touch DB index and ignore only-needed flag, false otherwise</param> /// TODO: Add ability to update .romba files with proper size AND use the correct depot if it fills up /// TODO: Add ability correctly to mark which depot the files are being rebuilt to in the DB private static void InitArchive( List <string> inputs, bool onlyNeeded, string resume, int includeZips, int workers, int includeGZips, int include7Zips, bool skipInitialScan, bool useGolangZip, // Obsolete bool noDb) { // First we want to get just all directories from the inputs List <string> onlyDirs = new List <string>(); foreach (string input in inputs) { if (Directory.Exists(input)) { onlyDirs.Add(Path.GetFullPath(input)); } } // Then process all of the input directories into an internal DAT DatFile df = new DatFile(); foreach (string dir in onlyDirs) { // TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually df.PopulateFromDir(dir, Hash.DeepHashes, false, false, SkipFileType.None, false, false, _tmpdir, false, null, true, null); df.PopulateFromDir(dir, Hash.DeepHashes, false, true, SkipFileType.None, false, false, _tmpdir, false, null, true, null); } // Create an empty Dat for files that need to be rebuilt DatFile need = new DatFile(); // Open the database connection SqliteConnection dbc = new SqliteConnection(_connectionString); dbc.Open(); // Now that we have the Dats, add the files to the database string crcquery = "INSERT OR IGNORE INTO crc (crc) VALUES"; string md5query = "INSERT OR IGNORE INTO md5 (md5) VALUES"; string sha1query = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES"; string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES"; string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES"; foreach (string key in df.Keys) { List <DatItem> datItems = df[key]; foreach (Rom rom in datItems) { // If we care about if the file exists, check the databse first if (onlyNeeded && !noDb) { string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1" + " WHERE crcsha1.crc=\"" + rom.CRC + "\"" + " OR md5sha1.md5=\"" + rom.MD5 + "\"" + " OR md5sha1.sha1=\"" + rom.SHA1 + "\""; SqliteCommand slc = new SqliteCommand(query, dbc); SqliteDataReader sldr = slc.ExecuteReader(); if (sldr.HasRows) { // Add to the queries if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcquery += " (\"" + rom.CRC + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5query += " (\"" + rom.MD5 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += " (\"" + rom.SHA1 + "\", \"" + _depots.Keys.ToList()[0] + "\"),"; if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),"; } } // Add to the Dat need.Add(key, rom); } } // Otherwise, just add the file to the list else { // Add to the queries if (!noDb) { if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcquery += " (\"" + rom.CRC + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5query += " (\"" + rom.MD5 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.SHA1)) { sha1query += " (\"" + rom.SHA1 + "\", \"" + _depots.Keys.ToList()[0] + "\"),"; if (!String.IsNullOrWhiteSpace(rom.CRC)) { crcsha1query += " (\"" + rom.CRC + "\", \"" + rom.SHA1 + "\"),"; } if (!String.IsNullOrWhiteSpace(rom.MD5)) { md5sha1query += " (\"" + rom.MD5 + "\", \"" + rom.SHA1 + "\"),"; } } } // Add to the Dat need.Add(key, rom); } } } // Now run the queries, if they're populated if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES") { SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES") { SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES") { SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES") { SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES") { SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc); slc.ExecuteNonQuery(); slc.Dispose(); } // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = Utilities.GetArchiveScanLevelFromNumbers(include7Zips, includeGZips, 2, includeZips); need.RebuildGeneric(onlyDirs, _depots.Keys.ToList()[0], false /*quickScan*/, false /*date*/, false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzip, true /*romba*/, asl, false /*updateDat*/, null /*headerToCheckAgainst*/, true /* chdsAsFiles */); }