Exemplo n.º 1
0
        public override bool ProcessFeatures(Dictionary <string, Feature> features)
        {
            // If the base fails, just fail out
            if (!base.ProcessFeatures(features))
            {
                return(false);
            }

            // Get feature flags
            string name        = GetString(features, NameStringValue);
            string description = GetString(features, DescriptionStringValue);
            string source      = GetString(features, SourceStringValue);
            string outdat      = GetString(features, OutStringValue);

            // Ensure the output directory
            outdat.Ensure(create: true);

            // Check that all required directories exist
            if (!Directory.Exists(source))
            {
                logger.Error($"File '{source}' does not exist!");
                return(false);
            }

            // Create and write the encapsulating datfile
            DatFile datfile = DatFile.Create();

            datfile.Header.Name        = string.IsNullOrWhiteSpace(name) ? "untitled" : name;
            datfile.Header.Description = description;
            DatFromDir.PopulateFromDir(datfile, source, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
            Writer.Write(datfile, outdat);
            return(true);
        }
Exemplo n.º 2
0
        public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
        {
            // If the base fails, just fail out
            if (!base.ProcessFeatures(features))
            {
                return(false);
            }

            // Get the archive scanning level
            // TODO: Remove usage
            int sevenzip = GetInt32(features, Include7ZipsInt32Value);
            int gz       = GetInt32(features, IncludeGZipsInt32Value);
            int zip      = GetInt32(features, IncludeZipsInt32Value);

            // Get feature flags
            bool noDb       = GetBoolean(features, NoDbValue);
            bool onlyNeeded = GetBoolean(features, OnlyNeededValue);

            // First we want to get just all directories from the inputs
            List <string> onlyDirs = new List <string>();

            foreach (string input in Inputs)
            {
                if (Directory.Exists(input))
                {
                    onlyDirs.Add(Path.GetFullPath(input));
                }
            }

            // Then process all of the input directories into an internal DAT
            DatFile df = DatFile.Create();

            foreach (string dir in onlyDirs)
            {
                DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
                DatFromDir.PopulateFromDir(df, dir, asFiles: TreatAsFile.All, hashes: Hash.Standard);
            }

            // Create an empty Dat for files that need to be rebuilt
            DatFile need = DatFile.Create();

            // Open the database connection
            SqliteConnection dbc = new SqliteConnection(_connectionString);

            dbc.Open();

            // Now that we have the Dats, add the files to the database
            string crcquery     = "INSERT OR IGNORE INTO crc (crc) VALUES";
            string md5query     = "INSERT OR IGNORE INTO md5 (md5) VALUES";
            string sha1query    = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
            string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
            string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";

            foreach (string key in df.Items.Keys)
            {
                ConcurrentList <DatItem> datItems = df.Items[key];
                foreach (Rom rom in datItems)
                {
                    // If we care about if the file exists, check the databse first
                    if (onlyNeeded && !noDb)
                    {
                        string query = "SELECT * FROM crcsha1 JOIN md5sha1 ON crcsha1.sha1=md5sha1.sha1"
                                       + $" WHERE crcsha1.crc=\"{rom.CRC}\""
                                       + $" OR md5sha1.md5=\"{rom.MD5}\""
                                       + $" OR md5sha1.sha1=\"{rom.SHA1}\"";
                        SqliteCommand    slc  = new SqliteCommand(query, dbc);
                        SqliteDataReader sldr = slc.ExecuteReader();

                        if (sldr.HasRows)
                        {
                            // Add to the queries
                            if (!string.IsNullOrWhiteSpace(rom.CRC))
                            {
                                crcquery += $" (\"{rom.CRC}\"),";
                            }

                            if (!string.IsNullOrWhiteSpace(rom.MD5))
                            {
                                md5query += $" (\"{rom.MD5}\"),";
                            }

                            if (!string.IsNullOrWhiteSpace(rom.SHA1))
                            {
                                sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";

                                if (!string.IsNullOrWhiteSpace(rom.CRC))
                                {
                                    crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
                                }

                                if (!string.IsNullOrWhiteSpace(rom.MD5))
                                {
                                    md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
                                }
                            }

                            // Add to the Dat
                            need.Items.Add(key, rom);
                        }
                    }
                    // Otherwise, just add the file to the list
                    else
                    {
                        // Add to the queries
                        if (!noDb)
                        {
                            if (!string.IsNullOrWhiteSpace(rom.CRC))
                            {
                                crcquery += $" (\"{rom.CRC}\"),";
                            }

                            if (!string.IsNullOrWhiteSpace(rom.MD5))
                            {
                                md5query += $" (\"{rom.MD5}\"),";
                            }

                            if (!string.IsNullOrWhiteSpace(rom.SHA1))
                            {
                                sha1query += $" (\"{rom.SHA1}\", \"{_depots.Keys.ToList()[0]}\"),";

                                if (!string.IsNullOrWhiteSpace(rom.CRC))
                                {
                                    crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
                                }

                                if (!string.IsNullOrWhiteSpace(rom.MD5))
                                {
                                    md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
                                }
                            }
                        }

                        // Add to the Dat
                        need.Items.Add(key, rom);
                    }
                }
            }

            // Now run the queries, if they're populated
            if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
            {
                SqliteCommand slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
                slc.ExecuteNonQuery();
                slc.Dispose();
            }

            if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
            {
                SqliteCommand slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
                slc.ExecuteNonQuery();
                slc.Dispose();
            }

            if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
            {
                SqliteCommand slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
                slc.ExecuteNonQuery();
                slc.Dispose();
            }

            if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
            {
                SqliteCommand slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
                slc.ExecuteNonQuery();
                slc.Dispose();
            }

            if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
            {
                SqliteCommand slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
                slc.ExecuteNonQuery();
                slc.Dispose();
            }

            // Create the sorting object to use and rebuild the needed files
            Rebuilder.RebuildGeneric(
                need,
                onlyDirs,
                outDir: _depots.Keys.ToList()[0],
                outputFormat: OutputFormat.TorrentGzipRomba,
                asFiles: TreatAsFile.NonArchive);

            return(true);
        }
Exemplo n.º 3
0
        public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
        {
            // If the base fails, just fail out
            if (!base.ProcessFeatures(features))
            {
                return(false);
            }

            logger.Error("This feature is not yet implemented: rescan-depots");

            foreach (string depotname in Inputs)
            {
                // Check that it's a valid depot first
                if (!_depots.ContainsKey(depotname))
                {
                    logger.User($"'{depotname}' is not a recognized depot. Please add it to your configuration file and try again");
                    return(false);
                }

                // Then check that the depot is online
                if (!Directory.Exists(depotname))
                {
                    logger.User($"'{depotname}' does not appear to be online. Please check its status and try again");
                    return(false);
                }

                // Open the database connection
                SqliteConnection dbc = new SqliteConnection(_connectionString);
                dbc.Open();

                // If we have it, then check for all hashes that are in that depot
                List <string>    hashes = new List <string>();
                string           query  = $"SELECT sha1 FROM sha1 WHERE depot=\"{depotname}\"";
                SqliteCommand    slc    = new SqliteCommand(query, dbc);
                SqliteDataReader sldr   = slc.ExecuteReader();
                if (sldr.HasRows)
                {
                    while (sldr.Read())
                    {
                        hashes.Add(sldr.GetString(0));
                    }
                }

                // Now rescan the depot itself
                DatFile depot = DatFile.Create();
                DatFromDir.PopulateFromDir(depot, depotname, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
                depot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);

                // Set the base queries to use
                string crcquery     = "INSERT OR IGNORE INTO crc (crc) VALUES";
                string md5query     = "INSERT OR IGNORE INTO md5 (md5) VALUES";
                string sha1query    = "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES";
                string crcsha1query = "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES";
                string md5sha1query = "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES";

                // Once we have both, check for any new files
                List <string>        dupehashes = new List <string>();
                IEnumerable <string> keys       = depot.Items.Keys;
                foreach (string key in keys)
                {
                    ConcurrentList <DatItem> roms = depot.Items[key];
                    foreach (Rom rom in roms)
                    {
                        if (hashes.Contains(rom.SHA1))
                        {
                            dupehashes.Add(rom.SHA1);
                            hashes.Remove(rom.SHA1);
                        }
                        else if (!dupehashes.Contains(rom.SHA1))
                        {
                            if (!string.IsNullOrWhiteSpace(rom.CRC))
                            {
                                crcquery += $" (\"{rom.CRC}\"),";
                            }

                            if (!string.IsNullOrWhiteSpace(rom.MD5))
                            {
                                md5query += $" (\"{rom.MD5}\"),";
                            }

                            if (!string.IsNullOrWhiteSpace(rom.SHA1))
                            {
                                sha1query += $" (\"{rom.SHA1}\", \"{depotname}\"),";

                                if (!string.IsNullOrWhiteSpace(rom.CRC))
                                {
                                    crcsha1query += $" (\"{rom.CRC}\", \"{rom.SHA1}\"),";
                                }

                                if (!string.IsNullOrWhiteSpace(rom.MD5))
                                {
                                    md5sha1query += $" (\"{rom.MD5}\", \"{rom.SHA1}\"),";
                                }
                            }
                        }
                    }
                }

                // Now run the queries after fixing them
                if (crcquery != "INSERT OR IGNORE INTO crc (crc) VALUES")
                {
                    slc = new SqliteCommand(crcquery.TrimEnd(','), dbc);
                    slc.ExecuteNonQuery();
                }

                if (md5query != "INSERT OR IGNORE INTO md5 (md5) VALUES")
                {
                    slc = new SqliteCommand(md5query.TrimEnd(','), dbc);
                    slc.ExecuteNonQuery();
                }

                if (sha1query != "INSERT OR IGNORE INTO sha1 (sha1, depot) VALUES")
                {
                    slc = new SqliteCommand(sha1query.TrimEnd(','), dbc);
                    slc.ExecuteNonQuery();
                }

                if (crcsha1query != "INSERT OR IGNORE INTO crcsha1 (crc, sha1) VALUES")
                {
                    slc = new SqliteCommand(crcsha1query.TrimEnd(','), dbc);
                    slc.ExecuteNonQuery();
                }

                if (md5sha1query != "INSERT OR IGNORE INTO md5sha1 (md5, sha1) VALUES")
                {
                    slc = new SqliteCommand(md5sha1query.TrimEnd(','), dbc);
                    slc.ExecuteNonQuery();
                }

                // Now that we've added the information, we get to remove all of the hashes that we want to
                query  = @"DELETE FROM sha1
JOIN crcsha1
    ON sha1.sha1=crcsha1.sha1
JOIN md5sha1
    ON sha1.sha1=md5sha1.sha1
JOIN crc
    ON crcsha1.crc=crc.crc
JOIN md5
    ON md5sha1.md5=md5.md5
WHERE sha1.sha1 IN ";
                query += $"({string.Join("\",\"", hashes)}\")";
                slc    = new SqliteCommand(query, dbc);
                slc.ExecuteNonQuery();

                // Dispose of the database connection
                slc.Dispose();
                dbc.Dispose();
            }

            return(true);
        }
Exemplo n.º 4
0
        public override bool ProcessFeatures(Dictionary <string, SabreTools.Help.Feature> features)
        {
            // If the base fails, just fail out
            if (!base.ProcessFeatures(features))
            {
                return(false);
            }

            // Get feature flags
            int    workers      = GetInt32(features, WorkersInt32Value);
            string missingSha1s = GetString(features, MissingSha1sStringValue);

            // Make sure the db is set
            if (string.IsNullOrWhiteSpace(_db))
            {
                _db = "db.sqlite";
                _connectionString = $"Data Source={_db};Version = 3;";
            }

            // Make sure the file exists
            if (!File.Exists(_db))
            {
                EnsureDatabase(_db, _connectionString);
            }

            // Make sure the dats dir is set
            if (string.IsNullOrWhiteSpace(_dats))
            {
                _dats = "dats";
            }

            _dats = Path.Combine(PathTool.GetRuntimeDirectory(), _dats);

            // Make sure the folder exists
            if (!Directory.Exists(_dats))
            {
                Directory.CreateDirectory(_dats);
            }

            // First get a list of SHA-1's from the input DATs
            DatFile datroot = DatFile.Create();

            datroot.Header.Type = "SuperDAT";
            DatFromDir.PopulateFromDir(datroot, _dats, asFiles: TreatAsFile.NonArchive, hashes: Hash.Standard);
            datroot.Items.BucketBy(ItemKey.SHA1, DedupeType.None);

            // Create a List of dat hashes in the database (SHA-1)
            List <string> databaseDats = new List <string>();
            List <string> unneeded     = new List <string>();

            SqliteConnection dbc = new SqliteConnection(_connectionString);

            dbc.Open();

            // Populate the List from the database
            InternalStopwatch watch = new InternalStopwatch("Populating the list of existing DATs");

            string           query = "SELECT DISTINCT hash FROM dat";
            SqliteCommand    slc   = new SqliteCommand(query, dbc);
            SqliteDataReader sldr  = slc.ExecuteReader();

            if (sldr.HasRows)
            {
                sldr.Read();
                string hash = sldr.GetString(0);
                if (datroot.Items.ContainsKey(hash))
                {
                    datroot.Items.Remove(hash);
                    databaseDats.Add(hash);
                }
                else if (!databaseDats.Contains(hash))
                {
                    unneeded.Add(hash);
                }
            }

            datroot.Items.BucketBy(ItemKey.Machine, DedupeType.None, norename: true);

            watch.Stop();

            slc.Dispose();
            sldr.Dispose();

            // Loop through the Dictionary and add all data
            watch.Start("Adding new DAT information");
            foreach (string key in datroot.Items.Keys)
            {
                foreach (Rom value in datroot.Items[key])
                {
                    AddDatToDatabase(value, dbc);
                }
            }

            watch.Stop();

            // Now loop through and remove all references to old Dats
            if (unneeded.Count > 0)
            {
                watch.Start("Removing unmatched DAT information");

                query = "DELETE FROM dat WHERE";
                foreach (string dathash in unneeded)
                {
                    query += $" OR hash=\"{dathash}\"";
                }

                query = query.Replace("WHERE OR", "WHERE");
                slc   = new SqliteCommand(query, dbc);
                slc.ExecuteNonQuery();
                slc.Dispose();

                watch.Stop();
            }

            dbc.Dispose();
            return(true);
        }