public BackendManager(string backendurl, Options options, IBackendWriter statwriter, LocalDatabase database) { m_options = options; m_backendurl = backendurl; m_statwriter = statwriter; m_taskControl = statwriter as BasicResults; m_db = new DatabaseCollector(database, statwriter); m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); if (m_backend == null) throw new Exception(string.Format("Backend not supported: {0}", m_backendurl)); if (!m_options.NoEncryption) { m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); if (m_encryption == null) throw new Exception(string.Format("Encryption method not supported: ", m_options.EncryptionModule)); } if (m_taskControl != null) m_taskControl.StateChangedEvent += (state) => { if (state == TaskControlState.Abort) m_thread.Abort(); }; m_queue = new BlockingQueue<FileEntryItem>(options.SynchronousUpload ? 1 : (options.AsynchronousUploadLimit == 0 ? int.MaxValue : options.AsynchronousUploadLimit)); m_thread = new System.Threading.Thread(this.ThreadRun); m_thread.Name = "Backend Async Worker"; m_thread.IsBackground = true; m_thread.Start(); }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> public static void VerifyLocalList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var locallist = database.GetRemoteVolumes(); foreach(var i in locallist) { switch (i.State) { case RemoteVolumeState.Uploaded: case RemoteVolumeState.Verified: case RemoteVolumeState.Deleted: break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: case RemoteVolumeState.Uploading: log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); try { backend.Delete(i.Name, i.Size, true); } catch (Exception ex) { log.AddWarning(string.Format("Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message), ex); } break; default: log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null); break; } backend.FlushDbMessages(); } }
public void DeleteLocalFile(IBackendWriter stat) { if (this.LocalTempfile != null) { try { this.LocalTempfile.Dispose(); } catch (Exception ex) { stat.AddWarning(string.Format("Failed to dispose temporary file: {0}", this.LocalTempfile), ex); } }
public DatabaseCollector(LocalDatabase database, IBackendWriter stats) { m_database = database; m_stats = stats; m_dbqueue = new List <IDbEntry>(); if (m_database != null) { m_callerThread = System.Threading.Thread.CurrentThread; } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var tp = RemoteListAnalysis(backend, options, database, log); long extraCount = 0; long missingCount = 0; foreach(var n in tp.ExtraVolumes) { log.AddWarning(string.Format("Extra unknown file: {0}", n.File.Name), null); extraCount++; } foreach(var n in tp.MissingVolumes) { log.AddWarning(string.Format("Missing file: {0}", n.Name), null); missingCount++; } if (extraCount > 0) { var s = string.Format("Found {0} remote files that are not recorded in local storage, please run repair", extraCount); log.AddError(s, null); throw new Exception(s); } var lookup = new Dictionary<string, string>(); var doubles = new Dictionary<string, string>(); foreach(var v in tp.ParsedVolumes) { if (lookup.ContainsKey(v.File.Name)) doubles[v.File.Name] = null; else lookup[v.File.Name] = null; } if (doubles.Count > 0) { var s = string.Format("Found remote files reported as duplicates, either the backend module is broken or you need to manually remove the extra copies.\nThe following files were found multiple times: {0}", string.Join(", ", doubles.Keys)); log.AddError(s, null); throw new Exception(s); } if (missingCount > 0) { string s; if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0) s = string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes)); else s = string.Format("Found {0} files that are missing from the remote storage, please run repair", missingCount); log.AddError(s, null); throw new Exception(s); } }
public void Encrypt(Library.Interface.IEncryption encryption, IBackendWriter stat) { if (encryption != null && !this.Encrypted) { var tempfile = new Library.Utility.TempFile(); encryption.Encrypt(this.LocalFilename, tempfile); this.DeleteLocalFile(stat); this.LocalTempfile = tempfile; this.Hash = null; this.Size = 0; this.Encrypted = true; } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var tp = RemoteListAnalysis(backend, options, database, log); long extraCount = 0; long missingCount = 0; foreach(var n in tp.ExtraVolumes) { log.AddWarning(string.Format("Extra unknown file: {0}", n.File.Name), null); extraCount++; } foreach(var n in tp.MissingVolumes) { log.AddWarning(string.Format("Missing file: {0}", n.Name), null); missingCount++; } if (extraCount > 0) { var s = string.Format("Found {0} remote files that are not recorded in local storage, please run repair", extraCount); log.AddError(s, null); throw new Exception(s); } if (missingCount > 0) { string s; if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0) s = string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes)); else s = string.Format("Found {0} files that are missing from the remote storage, please run repair", missingCount); log.AddError(s, null); throw new Exception(s); } }
/// <summary> /// Uploads the verification file. /// </summary> /// <param name="backendurl">The backend url</param> /// <param name="options">The options to use</param> /// <param name="result">The result writer</param> /// <param name="db">The attached database</param> /// <param name="transaction">An optional transaction object</param> public static void UploadVerificationFile(string backendurl, Options options, IBackendWriter result, LocalDatabase db, System.Data.IDbTransaction transaction) { using(var backend = new BackendManager(backendurl, options, result, db)) using(var tempfile = new Library.Utility.TempFile()) { var remotename = options.Prefix + "-verification.json"; using(var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) FilelistProcessor.CreateVerificationFile(db, stream); if (options.Dryrun) { result.AddDryrunMessage(string.Format("Would upload verification file: {0}, size: {1}", remotename, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(tempfile).Length))); } else { backend.PutUnencrypted(remotename, tempfile); backend.WaitForComplete(db, transaction); } } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="protectedfile">A filename that should be excempted for deletion</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, string protectedfile) { var rawlist = backend.List(); var lookup = new Dictionary <string, Volumes.IParsedVolume>(); protectedfile = protectedfile ?? string.Empty; var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix == options.Prefix select p).ToList(); var otherlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix != options.Prefix select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count; log.KnownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum(); log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); log.BackupListCount = filesets.Count; log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); // TODO: We should query through the backendmanager using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) if (bk is Library.Interface.IQuotaEnabledBackend) { Library.Interface.IQuotaInfo quota = ((Library.Interface.IQuotaEnabledBackend)bk).Quota; if (quota != null) { log.TotalQuotaSpace = quota.TotalQuotaSpace; log.FreeQuotaSpace = quota.FreeQuotaSpace; } } log.AssignedQuotaSpace = options.QuotaSize; foreach (var s in remotelist) { lookup[s.File.Name] = s; } var missing = new List <RemoteVolumeEntry>(); var missingHash = new List <Tuple <long, RemoteVolumeEntry> >(); var cleanupRemovedRemoteVolumes = new HashSet <string>(); foreach (var e in database.DuplicateRemoteVolumes()) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) { database.UnlinkRemoteVolume(e.Key, e.Value); } else { throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString())); } } var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { Volumes.IParsedVolume r; var remoteFound = lookup.TryGetValue(i.Name, out r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); lookup.Remove(i.Name); switch (i.State) { case RemoteVolumeState.Deleted: if (remoteFound) { log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name)); } break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: if (remoteFound) { log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } else { if (i.deleteGracePeriod > DateTime.UtcNow) { log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.deleteGracePeriod.ToLocalTime())); } else { if (string.Equals(i.Name, protectedfile) && i.State == RemoteVolumeState.Temporary) { log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name)); } else { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); cleanupRemovedRemoteVolumes.Add(i.Name); } } } break; case RemoteVolumeState.Uploading: if (remoteFound && correctSize && r.File.Size >= 0) { log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded)); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else if (!remoteFound) { if (string.Equals(i.Name, protectedfile)) { log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name)); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); } else { log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name)); cleanupRemovedRemoteVolumes.Add(i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); } } else { if (string.Equals(i.Name, protectedfile)) { log.AddMessage(string.Format("keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name)); } else { log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } } break; case RemoteVolumeState.Uploaded: if (!remoteFound) { missing.Add(i); } else if (correctSize) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } else { missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i)); } break; case RemoteVolumeState.Verified: if (!remoteFound) { missing.Add(i); } else if (!correctSize) { missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i)); } break; default: log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null); break; } backend.FlushDbMessages(); } // cleanup deleted volumes in DB en block database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null); foreach (var i in missingHash) { log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null); } return(new RemoteAnalysisResult() { ParsedVolumes = remotelist, OtherVolumes = otherlist, ExtraVolumes = lookup.Values, MissingVolumes = missing, VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }); }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var rawlist = backend.List(); var lookup = new Dictionary<string, Volumes.IParsedVolume>(); var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix == options.Prefix select p).ToList(); var otherlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix != options.Prefix select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count(); log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum(); log.UnknownFileCount = unknownlist.Count(); log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum(); log.BackupListCount = filesets.Count; log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); if (backend is Library.Interface.IQuotaEnabledBackend) { log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace; log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace; } log.AssignedQuotaSpace = options.QuotaSize; foreach(var s in remotelist) lookup[s.File.Name] = s; var missing = new List<RemoteVolumeEntry>(); var missingHash = new List<Tuple<long, RemoteVolumeEntry>>(); var cleanupRemovedRemoteVolumes = new HashSet<string>(); foreach(var e in database.DuplicateRemoteVolumes()) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) database.UnlinkRemoteVolume(e.Key, e.Value); else throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString())); } var locallist = database.GetRemoteVolumes(); foreach(var i in locallist) { Volumes.IParsedVolume r; var remoteFound = lookup.TryGetValue(i.Name, out r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); lookup.Remove(i.Name); switch (i.State) { case RemoteVolumeState.Deleted: if (remoteFound) log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name)); break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: if (remoteFound) { log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } else { if (i.deleteGracePeriod > DateTime.UtcNow) { log.AddMessage(string.Format("keeping delete request for {0} until {1}", i.Name, i.deleteGracePeriod.ToLocalTime())); } else { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); cleanupRemovedRemoteVolumes.Add(i.Name); } } break; case RemoteVolumeState.Uploading: if (remoteFound && correctSize && r.File.Size >= 0) { log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded)); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else if (!remoteFound) { log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name)); cleanupRemovedRemoteVolumes.Add(i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); } else { log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } break; case RemoteVolumeState.Uploaded: if (!remoteFound) missing.Add(i); else if (correctSize) database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); else missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i)); break; case RemoteVolumeState.Verified: if (!remoteFound) missing.Add(i); else if (!correctSize) missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i)); break; default: log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null); break; } backend.FlushDbMessages(); } // cleanup deleted volumes in DB en block database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null); foreach(var i in missingHash) log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null); return new RemoteAnalysisResult() { ParsedVolumes = remotelist, OtherVolumes = otherlist, ExtraVolumes = lookup.Values, MissingVolumes = missing, VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }; }
public StatsCollector(IBackendWriter bw) { m_bw = bw; }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> public static void VerifyLocalList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { switch (i.State) { case RemoteVolumeState.Uploaded: case RemoteVolumeState.Verified: case RemoteVolumeState.Deleted: break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: case RemoteVolumeState.Uploading: log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); try { backend.Delete(i.Name, i.Size, true); } catch (Exception ex) { log.AddWarning(string.Format("Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message), ex); } break; default: log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null); break; } backend.FlushDbMessages(); } }
/// <summary> /// Uploads the verification file. /// </summary> /// <param name="backendurl">The backend url</param> /// <param name="options">The options to use</param> /// <param name="result">The result writer</param> /// <param name="db">The attached database</param> /// <param name="transaction">An optional transaction object</param> public static void UploadVerificationFile(string backendurl, Options options, IBackendWriter result, LocalDatabase db, System.Data.IDbTransaction transaction) { using (var backend = new BackendManager(backendurl, options, result, db)) using (var tempfile = new Library.Utility.TempFile()) { var remotename = options.Prefix + "-verification.json"; using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) FilelistProcessor.CreateVerificationFile(db, stream); if (options.Dryrun) { result.AddDryrunMessage(string.Format("Would upload verification file: {0}, size: {1}", remotename, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(tempfile).Length))); } else { backend.PutUnencrypted(remotename, tempfile); backend.WaitForComplete(db, transaction); } } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> /// <param name="protectedFiles">Filenames that should be exempted from deletion</param> public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable <string> protectedFiles = null) { var tp = RemoteListAnalysis(backend, options, database, log, protectedFiles); long extraCount = 0; long missingCount = 0; foreach (var n in tp.ExtraVolumes) { Logging.Log.WriteWarningMessage(LOGTAG, "ExtraUnknownFile", null, "Extra unknown file: {0}", n.File.Name); extraCount++; } foreach (var n in tp.MissingVolumes) { Logging.Log.WriteWarningMessage(LOGTAG, "MissingFile", null, "Missing file: {0}", n.Name); missingCount++; } if (extraCount > 0) { var s = string.Format("Found {0} remote files that are not recorded in local storage, please run repair", extraCount); Logging.Log.WriteErrorMessage(LOGTAG, "ExtraRemoteFiles", null, s); throw new Duplicati.Library.Interface.UserInformationException(s, "ExtraRemoteFiles"); } ISet <string> doubles; Library.Utility.Utility.GetUniqueItems(tp.ParsedVolumes.Select(x => x.File.Name), out doubles); if (doubles.Count > 0) { var s = string.Format("Found remote files reported as duplicates, either the backend module is broken or you need to manually remove the extra copies.\nThe following files were found multiple times: {0}", string.Join(", ", doubles)); Logging.Log.WriteErrorMessage(LOGTAG, "DuplicateRemoteFiles", null, s); throw new Duplicati.Library.Interface.UserInformationException(s, "DuplicateRemoteFiles"); } if (missingCount > 0) { string s; if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0) { s = string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes)); } else { s = string.Format("Found {0} files that are missing from the remote storage, please run repair", missingCount); } Logging.Log.WriteErrorMessage(LOGTAG, "MissingRemoteFiles", null, s); throw new Duplicati.Library.Interface.UserInformationException(s, "MissingRemoteFiles"); } }
public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, IDbTransaction transaction) { if (!options.NoBackendverification) { LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); IEnumerable <string> protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); FilelistProcessor.VerifyRemoteList(backend, options, database, backendWriter, protectedFiles); } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var rawlist = backend.List(); var lookup = new Dictionary<string, Volumes.IParsedVolume>(); var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count(); log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum(); log.UnknownFileCount = unknownlist.Count(); log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum(); log.BackupListCount = filesets.Count; log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); if (backend is Library.Interface.IQuotaEnabledBackend) { log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace; log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace; } log.AssignedQuotaSpace = options.QuotaSize; foreach(var s in remotelist) if (s.Prefix == options.Prefix) lookup[s.File.Name] = s; var missing = new List<RemoteVolumeEntry>(); var missingHash = new List<Tuple<long, RemoteVolumeEntry>>(); var locallist = database.GetRemoteVolumes(); foreach(var i in locallist) { Volumes.IParsedVolume r; var remoteFound = lookup.TryGetValue(i.Name, out r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); lookup.Remove(i.Name); switch (i.State) { case RemoteVolumeState.Deleted: if (remoteFound) log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name)); break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: if (remoteFound) { log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } else { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } break; case RemoteVolumeState.Uploading: if (remoteFound && correctSize && r.File.Size >= 0) { log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded)); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else { log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } break; case RemoteVolumeState.Uploaded: if (!remoteFound) missing.Add(i); else if (correctSize) database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); else missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i)); break; case RemoteVolumeState.Verified: if (!remoteFound) missing.Add(i); else if (!correctSize) missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i)); break; default: log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null); break; } } foreach(var i in missingHash) log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null); return new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing, VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }; }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var rawlist = backend.List(); var lookup = new Dictionary <string, Volumes.IParsedVolume>(); var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count(); log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum(); log.UnknownFileCount = unknownlist.Count(); log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum(); log.BackupListCount = filesets.Count; log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); if (backend is Library.Interface.IQuotaEnabledBackend) { log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace; log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace; } log.AssignedQuotaSpace = options.QuotaSize; foreach (var s in remotelist) { if (s.Prefix == options.Prefix) { lookup[s.File.Name] = s; } } var missing = new List <RemoteVolumeEntry>(); var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { //Ignore those that are deleted if (i.State == RemoteVolumeState.Deleted) { continue; } if (i.State == RemoteVolumeState.Temporary) { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } else if (i.State == RemoteVolumeState.Deleting && lookup.ContainsKey(i.Name)) { log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); lookup.Remove(i.Name); } else { Volumes.IParsedVolume r; if (!lookup.TryGetValue(i.Name, out r)) { if (i.State == RemoteVolumeState.Uploading || i.State == RemoteVolumeState.Deleting || (r != null && r.File.Size != i.Size && r.File.Size >= 0 && i.Size >= 0)) { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } else { missing.Add(i); } } else if (i.State != RemoteVolumeState.Verified) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } lookup.Remove(i.Name); } } return(new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing }); }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var rawlist = backend.List(); var lookup = new Dictionary<string, Volumes.IParsedVolume>(); var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count(); log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum(); log.UnknownFileCount = unknownlist.Count(); log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum(); log.BackupListCount = filesets.Count; log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); if (backend is Library.Interface.IQuotaEnabledBackend) { log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace; log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace; } log.AssignedQuotaSpace = options.QuotaSize; foreach (var s in remotelist) if (s.Prefix == options.Prefix) lookup[s.File.Name] = s; var missing = new List<RemoteVolumeEntry>(); var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { //Ignore those that are deleted if (i.State == RemoteVolumeState.Deleted) continue; if (i.State == RemoteVolumeState.Temporary) { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } else if (i.State == RemoteVolumeState.Deleting && lookup.ContainsKey(i.Name)) { log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); lookup.Remove(i.Name); } else { Volumes.IParsedVolume r; if (!lookup.TryGetValue(i.Name, out r)) { if (i.State == RemoteVolumeState.Uploading || i.State == RemoteVolumeState.Deleting || (r != null && r.File.Size != i.Size && r.File.Size >= 0 && i.Size >= 0)) { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } else missing.Add(i); } else if (i.State != RemoteVolumeState.Verified) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } lookup.Remove(i.Name); } } return new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing }; }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="protectedFiles">Filenames that should be exempted from deletion</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable <string> protectedFiles) { var rawlist = backend.List(); var lookup = new Dictionary <string, Volumes.IParsedVolume>(); protectedFiles = protectedFiles ?? Enumerable.Empty <string>(); var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix == options.Prefix select p).ToList(); var otherlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix != options.Prefix select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count; long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum(); log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); log.BackupListCount = database.FilesetTimes.Count(); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); // TODO: We should query through the backendmanager using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) if (bk is IQuotaEnabledBackend enabledBackend) { Library.Interface.IQuotaInfo quota = enabledBackend.Quota; if (quota != null) { log.TotalQuotaSpace = quota.TotalQuotaSpace; log.FreeQuotaSpace = quota.FreeQuotaSpace; // Check to see if there should be a warning or error about the quota // Since this processor may be called multiple times during a backup // (both at the start and end, for example), the log keeps track of // whether a quota error or warning has been sent already. // Note that an error can still be sent later even if a warning was sent earlier. if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0) { log.ReportedQuotaError = true; Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); } else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info { // Warnings are sent if the available free space is less than the given percentage of the total backup size. double warningThreshold = options.QuotaWarningThreshold / (double)100; if (quota.FreeQuotaSpace < warningThreshold * knownFileSize) { log.ReportedQuotaWarning = true; Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); } } } } log.AssignedQuotaSpace = options.QuotaSize; foreach (var s in remotelist) { lookup[s.File.Name] = s; } var missing = new List <RemoteVolumeEntry>(); var missingHash = new List <Tuple <long, RemoteVolumeEntry> >(); var cleanupRemovedRemoteVolumes = new HashSet <string>(); foreach (var e in database.DuplicateRemoteVolumes()) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) { database.UnlinkRemoteVolume(e.Key, e.Value); } else { throw new Exception(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString())); } } var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { Volumes.IParsedVolume r; var remoteFound = lookup.TryGetValue(i.Name, out r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); lookup.Remove(i.Name); switch (i.State) { case RemoteVolumeState.Deleted: if (remoteFound) { Logging.Log.WriteInformationMessage(LOGTAG, "IgnoreRemoteDeletedFile", "ignoring remote file listed as {0}: {1}", i.State, i.Name); } break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: if (remoteFound) { Logging.Log.WriteInformationMessage(LOGTAG, "RemoveUnwantedRemoteFile", "removing remote file listed as {0}: {1}", i.State, i.Name); backend.Delete(i.Name, i.Size, true); } else { if (i.DeleteGracePeriod > DateTime.UtcNow) { Logging.Log.WriteInformationMessage(LOGTAG, "KeepDeleteRequest", "keeping delete request for {0} until {1}", i.Name, i.DeleteGracePeriod.ToLocalTime()); } else { if (i.State == RemoteVolumeState.Temporary && protectedFiles.Any(pf => pf == i.Name)) { Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); } else { Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "removing file listed as {0}: {1}", i.State, i.Name); cleanupRemovedRemoteVolumes.Add(i.Name); } } } break; case RemoteVolumeState.Uploading: if (remoteFound && correctSize && r.File.Size >= 0) { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else if (!remoteFound) { if (protectedFiles.Any(pf => pf == i.Name)) { Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null); } else { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); cleanupRemovedRemoteVolumes.Add(i.Name); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null); } } else { if (protectedFiles.Any(pf => pf == i.Name)) { Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); } else { Logging.Log.WriteInformationMessage(LOGTAG, "Remove incomplete file", "removing incomplete remote file listed as {0}: {1}", i.State, i.Name); backend.Delete(i.Name, i.Size, true); } } break; case RemoteVolumeState.Uploaded: if (!remoteFound) { missing.Add(i); } else if (correctSize) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } else { missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i)); } break; case RemoteVolumeState.Verified: if (!remoteFound) { missing.Add(i); } else if (!correctSize) { missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i)); } break; default: Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "unknown state for remote file listed as {0}: {1}", i.State, i.Name); break; } backend.FlushDbMessages(); } // cleanup deleted volumes in DB en block database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null); foreach (var i in missingHash) { Logging.Log.WriteWarningMessage(LOGTAG, "MissingRemoteHash", null, "remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash); } return(new RemoteAnalysisResult() { ParsedVolumes = remotelist, OtherVolumes = otherlist, ExtraVolumes = lookup.Values, MissingVolumes = missing, VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }); }
public void DeleteLocalFile(IBackendWriter stat) { if (this.LocalTempfile != null) try { this.LocalTempfile.Dispose(); } catch (Exception ex) { stat.AddWarning(string.Format("Failed to dispose temporary file: {0}", this.LocalTempfile), ex); } finally { this.LocalTempfile = null; } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var tp = RemoteListAnalysis(backend, options, database, log); long extraCount = 0; long missingCount = 0; foreach (var n in tp.ExtraVolumes) { log.AddWarning(string.Format("Extra unknown file: {0}", n.File.Name), null); extraCount++; } foreach (var n in tp.MissingVolumes) { log.AddWarning(string.Format("Missing file: {0}", n.Name), null); missingCount++; } if (extraCount > 0) { var s = string.Format("Found {0} remote files that are not recorded in local storage, please run repair", extraCount); log.AddError(s, null); throw new Exception(s); } var lookup = new Dictionary <string, string>(); var doubles = new Dictionary <string, string>(); foreach (var v in tp.ParsedVolumes) { if (lookup.ContainsKey(v.File.Name)) { doubles[v.File.Name] = null; } else { lookup[v.File.Name] = null; } } if (doubles.Count > 0) { var s = string.Format("Found remote files reported as duplicates, either the backend module is broken or you need to manually remove the extra copies.\nThe following files were found multiple times: ", string.Join(", ", doubles.Keys)); log.AddError(s, null); throw new Exception(s); } if (missingCount > 0) { string s; if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0) { s = string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes)); } else { s = string.Format("Found {0} files that are missing from the remote storage, please run repair", missingCount); } log.AddError(s, null); throw new Exception(s); } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> public static RemoteAnalysisResult RemoteListAnalysis(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var rawlist = backend.List(); var lookup = new Dictionary <string, Volumes.IParsedVolume>(); var remotelist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix == options.Prefix select p).ToList(); var otherlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null && p.Prefix != options.Prefix select p).ToList(); var unknownlist = (from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p == null select n).ToList(); var filesets = (from n in remotelist where n.FileType == RemoteVolumeType.Files orderby n.Time descending select n).ToList(); log.KnownFileCount = remotelist.Count(); log.KnownFileSize = remotelist.Select(x => x.File.Size).Sum(); log.UnknownFileCount = unknownlist.Count(); log.UnknownFileSize = unknownlist.Select(x => x.Size).Sum(); log.BackupListCount = filesets.Count; log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); if (backend is Library.Interface.IQuotaEnabledBackend) { log.TotalQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).TotalQuotaSpace; log.FreeQuotaSpace = ((Library.Interface.IQuotaEnabledBackend)backend).FreeQuotaSpace; } log.AssignedQuotaSpace = options.QuotaSize; foreach (var s in remotelist) { lookup[s.File.Name] = s; } var missing = new List <RemoteVolumeEntry>(); var missingHash = new List <Tuple <long, RemoteVolumeEntry> >(); var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { Volumes.IParsedVolume r; var remoteFound = lookup.TryGetValue(i.Name, out r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); lookup.Remove(i.Name); switch (i.State) { case RemoteVolumeState.Deleted: if (remoteFound) { log.AddMessage(string.Format("ignoring remote file listed as {0}: {1}", i.State, i.Name)); } break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: if (remoteFound) { log.AddMessage(string.Format("removing remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } else { log.AddMessage(string.Format("removing file listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); } break; case RemoteVolumeState.Uploading: if (remoteFound && correctSize && r.File.Size >= 0) { log.AddMessage(string.Format("promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded)); database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); } else if (!remoteFound) { log.AddMessage(string.Format("scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name)); database.RemoveRemoteVolume(i.Name, null); database.RegisterRemoteVolume(i.Name, i.Type, RemoteVolumeState.Deleting, null); } else { log.AddMessage(string.Format("removing incomplete remote file listed as {0}: {1}", i.State, i.Name)); backend.Delete(i.Name, i.Size, true); } break; case RemoteVolumeState.Uploaded: if (!remoteFound) { missing.Add(i); } else if (correctSize) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } else { missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i)); } break; case RemoteVolumeState.Verified: if (!remoteFound) { missing.Add(i); } else if (!correctSize) { missingHash.Add(new Tuple <long, RemoteVolumeEntry>(r.File.Size, i)); } break; default: log.AddWarning(string.Format("unknown state for remote file listed as {0}: {1}", i.State, i.Name), null); break; } backend.FlushDbMessages(); } foreach (var i in missingHash) { log.AddWarning(string.Format("remote file {1} is listed as {0} with size {2} but should be {3}, please verify the sha256 hash \"{4}\"", i.Item2.State, i.Item2.Name, i.Item1, i.Item2.Size, i.Item2.Hash), null); } return(new RemoteAnalysisResult() { ParsedVolumes = remotelist, OtherVolumes = otherlist, ExtraVolumes = lookup.Values, MissingVolumes = missing, VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }); }
public DatabaseCollector(LocalDatabase database, IBackendWriter stats) { m_database = database; m_stats = stats; m_dbqueue = new List<IDbEntry>(); if (m_database != null) m_callerThread = System.Threading.Thread.CurrentThread; }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="backend">The backend instance to use</param> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> public static void VerifyRemoteList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var tp = RemoteListAnalysis(backend, options, database, log); long extraCount = 0; long missingCount = 0; foreach (var n in tp.ExtraVolumes) { log.AddWarning(string.Format("Extra unknown file: {0}", n.File.Name), null); extraCount++; } foreach (var n in tp.MissingVolumes) { log.AddWarning(string.Format("Missing file: {0}", n.Name), null); missingCount++; } if (extraCount > 0) { var s = string.Format("Found {0} remote files that are not recorded in local storage, please run repair", extraCount); log.AddError(s, null); throw new Exception(s); } if (missingCount > 0) { string s; if (!tp.BackupPrefixes.Contains(options.Prefix) && tp.BackupPrefixes.Length > 0) { s = string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.Prefix, string.Join(", ", tp.BackupPrefixes)); } else { s = string.Format("Found {0} files that are missing from the remote storage, please run repair", missingCount); } log.AddError(s, null); throw new Exception(s); } }
/// <summary> /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// </summary> /// <param name="options">The options used</param> /// <param name="database">The database to compare with</param> /// <param name="log">The log instance to use</param> public static void VerifyLocalList(BackendManager backend, Options options, LocalDatabase database, IBackendWriter log) { var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { switch (i.State) { case RemoteVolumeState.Uploaded: case RemoteVolumeState.Verified: case RemoteVolumeState.Deleted: break; case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: case RemoteVolumeState.Uploading: Logging.Log.WriteInformationMessage(LOGTAG, "RemovingStaleFile", "Removing remote file listed as {0}: {1}", i.State, i.Name); try { backend.Delete(i.Name, i.Size, true); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DeleteFileFailed", ex, "Failed to erase file {0}, treating as deleted: {1}", i.Name, ex.Message); } break; default: Logging.Log.WriteWarningMessage(LOGTAG, "UnknownFileState", null, "Unknown state for remote file listed as {0}: {1}", i.State, i.Name); break; } backend.FlushDbMessages(); } }