public void Run(Library.Utility.IFilter filter, Func <long, DateTime, long, string, long, bool> callbackhandler = null) { if (!System.IO.File.Exists(m_options.Dbpath)) { throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); } using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath)) using (var tr = db.BeginTransaction()) DoRun(db, tr, filter, callbackhandler); }
public void Run(Library.Utility.IFilter filter) { if (!System.IO.File.Exists(m_options.Dbpath)) { throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath)); } if (filter != null && !filter.Empty) { throw new UserInformationException("Filters are not supported for this operation"); } List <Database.RemoteVolumeEntry> missing = null; using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath)) using (var tr = db.BeginTransaction()) { if (db.PartiallyRecreated) { throw new UserInformationException("The command does not work on partially recreated databases"); } var sets = ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, tr, m_options, out missing); if (sets == null) { return; } if (sets.Length == 0) { if (missing == null) { m_result.AddMessage("Found no broken filesets"); } else if (missing.Count == 0) { m_result.AddMessage("Found no broken filesets and no missing remote files"); } else { throw new UserInformationException(string.Format("Found no broken filesets, but {0} missing remote files", sets.Length)); } } m_result.AddMessage(string.Format("Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.Item3))); var pgoffset = 0.0f; var pgspan = 0.95f / sets.Length; var filesets = db.FilesetTimes.ToList(); var compare_list = sets.Select(x => new { FilesetID = x.Item2, Timestamp = x.Item1, RemoveCount = x.Item3, Version = filesets.FindIndex(y => y.Key == x.Item2), SetCount = db.GetFilesetFileCount(x.Item2, tr) }).ToArray(); var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); if (fully_emptied.Length != 0) { if (fully_emptied.Length == 1) { m_result.AddMessage(string.Format("Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount)); } else { m_result.AddMessage(string.Format("Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())))); } m_result.DeleteResults = new DeleteResults(m_result); using (var rmdb = new Database.LocalDeleteDatabase(db)) { var deltr = rmdb.BeginTransaction(); try { var opts = new Options(new Dictionary <string, string>(m_options.RawOptions)); opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; new DeleteHandler(m_backendurl, opts, (DeleteResults)m_result.DeleteResults) .DoRun(rmdb, ref deltr, true, false); if (!m_options.Dryrun) { using (new Logging.Timer("CommitDelete")) deltr.Commit(); rmdb.WriteResults(); } else { deltr.Rollback(); } } finally { if (deltr != null) { try { deltr.Rollback(); } catch { } } } } pgoffset += (pgspan * fully_emptied.Length); m_result.OperationProgressUpdater.UpdateProgress(pgoffset); } if (to_purge.Length > 0) { m_result.PurgeResults = new PurgeFilesResults(m_result); foreach (var bs in to_purge) { m_result.AddMessage(string.Format("Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime())); var opts = new Options(new Dictionary <string, string>(m_options.RawOptions)); using (var pgdb = new Database.LocalPurgeDatabase(db)) { // Recompute the version number after we deleted the versions before filesets = pgdb.FilesetTimes.ToList(); var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); if (thisversion < 0) { throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); } opts.RawOptions["version"] = thisversion.ToString(); opts.RawOptions.Remove("time"); opts.RawOptions["no-auto-compact"] = "true"; new PurgeFilesHandler(m_backendurl, opts, (PurgeFilesResults)m_result.PurgeResults).Run(pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => { if (filesetid != bs.FilesetID) { throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); } db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction); }); } pgoffset += pgspan; m_result.OperationProgressUpdater.UpdateProgress(pgoffset); } } if (m_options.Dryrun) { tr.Rollback(); } else { tr.Commit(); } m_result.OperationProgressUpdater.UpdateProgress(0.95f); if (missing != null && missing.Count > 0) { using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db)) { foreach (var f in missing) { if (m_options.Dryrun) { m_result.AddDryrunMessage(string.Format("Would delete remote file: {0}, size: {1}", f.Name, Library.Utility.Utility.FormatSizeString(f.Size))); } else { backend.Delete(f.Name, f.Size); } } } } if (!m_options.Dryrun && db.RepairInProgress) { m_result.AddMessage("Database was previously marked as in-progress, checking if it is valid after purging files"); db.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize, true); m_result.AddMessage("Purge completed, and consistency checks completed, marking database as complete"); db.RepairInProgress = false; } m_result.OperationProgressUpdater.UpdateProgress(1.0f); } }
private void DoRun(Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Library.Utility.IFilter filter, Func <long, DateTime, long, string, long, bool> callbackhandler) { if (filter != null && !filter.Empty) { throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); } if (db.PartiallyRecreated) { throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); } List <Database.RemoteVolumeEntry> missing; var brokensets = GetBrokenFilesetsFromRemote(m_backendurl, m_result, db, transaction, m_options, out missing); if (brokensets == null) { return; } if (brokensets.Length == 0) { m_result.BrokenFiles = new Tuple <long, DateTime, IEnumerable <Tuple <string, long> > > [0]; Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "No broken filesets found"); return; } var fstimes = db.FilesetTimes.ToList(); var brokenfilesets = brokensets.Select(x => new { Version = fstimes.FindIndex(y => y.Key == x.Item2), Timestamp = x.Item1, FilesetID = x.Item2, BrokenCount = x.Item3 } ) .ToArray(); m_result.BrokenFiles = brokenfilesets.Select( x => new Tuple <long, DateTime, IEnumerable <Tuple <string, long> > >( x.Version, x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly ? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable() : new MockList <Tuple <string, long> >((int)x.BrokenCount) )) .ToArray(); if (callbackhandler != null) { foreach (var bs in brokenfilesets) { foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, transaction)) { if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2)) { break; } } } } }
public static Tuple <DateTime, long, long>[] GetBrokenFilesetsFromRemote(string backendurl, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options, out List <Database.RemoteVolumeEntry> missing) { missing = null; var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); if (brokensets.Length == 0) { if (db.RepairInProgress) { throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair"); } Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); using (var backend = new BackendManager(backendurl, options, result.BackendWriter, db)) { var remotestate = FilelistProcessor.RemoteListAnalysis(backend, options, db, result.BackendWriter, null); if (!remotestate.ParsedVolumes.Any()) { throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); } missing = remotestate.MissingVolumes.ToList(); if (missing.Count == 0) { Logging.Log.WriteInformationMessage(LOGTAG, "NoMissingFilesFound", "Skipping operation because no files were found to be missing, and no filesets were recorded as broken."); return(null); } // Mark all volumes as disposable foreach (var f in missing) { db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); } Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); // Drop all content from tables db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); } brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); } return(brokensets); }