private string RemoveBackupSets(BackendWrapper backend, List<ManifestEntry> entries) { StringBuilder sb = new StringBuilder(); sb.Append(backend.FinishDeleteTransaction(false)); if (entries.Count > 0) { System.Xml.XmlDocument doc = new System.Xml.XmlDocument(); System.Xml.XmlNode root = doc.AppendChild(doc.CreateElement("files")); root.Attributes.Append(doc.CreateAttribute("version")).Value = "1"; foreach (ManifestEntry me in entries) { if (me.Alternate != null) root.AppendChild(doc.CreateElement("file")).InnerText = me.Alternate.Filename; root.AppendChild(doc.CreateElement("file")).InnerText = me.Filename; if (me.Verification != null) root.AppendChild(doc.CreateElement("file")).InnerText = me.Verification.Filename; foreach (KeyValuePair<SignatureEntry, ContentEntry> kx in me.Volumes) { root.AppendChild(doc.CreateElement("file")).InnerText = kx.Key.Filename; root.AppendChild(doc.CreateElement("file")).InnerText = kx.Value.Filename; } } if (m_options.Force) { using (TempFile tf = new TempFile()) { doc.Save(tf); tf.Protected = true; backend.WriteDeleteTransactionFile(tf); } } foreach (ManifestEntry me in entries) { sb.AppendLine(string.Format(Strings.Interface.DeletingBackupSetMessage, me.Time.ToString(System.Globalization.CultureInfo.InvariantCulture))); if (m_options.Force) { //Delete manifest if (me.Alternate != null) backend.Delete(me.Alternate); backend.Delete(me); if (me.Verification != null) backend.Delete(me.Verification); foreach (KeyValuePair<SignatureEntry, ContentEntry> kx in me.Volumes) { backend.Delete(kx.Key); backend.Delete(kx.Value); } } } if (m_options.Force) backend.RemoveDeleteTransactionFile(); if (!m_options.Force && entries.Count > 0) sb.AppendLine(Strings.Interface.FilesAreNotForceDeletedMessage); } return sb.ToString(); }
/// <summary> /// Downloads all required signature files from the backend. /// </summary> /// <param name="backend">The backend to read from</param> /// <param name="entries">The flattened list of manifests</param> /// <param name="tempfolder">The tempfolder set for this operation</param> /// <param name="allowHashFail">True to ignore files with failed hash signature</param> /// <returns>A list of file archives</returns> private List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> FindPatches(BackendWrapper backend, List<ManifestEntry> entries, string tempfolder, bool allowHashFail, CommunicationStatistics stat) { List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = new List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>>(); using (new Logging.Timer("Reading incremental data")) { OperationProgress(this, GetOperationType(), stat.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementalData, ""); //Calculate the total number of files to download //, and verify their order int incCount = 0; foreach (ManifestEntry be in entries) { int volNo = 0; //Prevent order based bugs if (entries.IndexOf(be) > 0) if (entries[entries.IndexOf(be) - 1].Time >= be.Time) throw new Exception(Strings.Interface.BadSortingDetectedError); incCount++; foreach (KeyValuePair<SignatureEntry, ContentEntry> bes in be.Volumes) { incCount++; if (volNo + 1 != bes.Key.Volumenumber || bes.Key.Volumenumber != bes.Value.Volumenumber) throw new Exception(Strings.Interface.BadVolumeSortOrder); volNo++; } } //The incremental part has a fixed cost, and each file has a fixed fraction of that double unitCost = m_incrementalFraction / incCount; //Ensure that the manifest chain has not been tampered with // since we will read all manifest files anyway, there is no harm in doing it here if (!m_options.DontReadManifests && entries.Count > 0) VerifyManifestChain(backend, entries[entries.Count - 1]); foreach (ManifestEntry be in entries) { m_progress += unitCost; Manifestfile manifest = GetManifest(backend, be); foreach (KeyValuePair<SignatureEntry, ContentEntry> bes in be.Volumes) { m_progress += unitCost; //Skip non-listed incrementals if (manifest.SignatureHashes != null && bes.Key.Volumenumber > manifest.SignatureHashes.Count) { backend.AddOrphan(bes.Key); backend.AddOrphan(bes.Value); continue; } OperationProgress(this, GetOperationType(), stat.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusReadingSignatureFile, be.Time.ToShortDateString() + " " + be.Time.ToShortTimeString(), bes.Key.Volumenumber), ""); string filename = System.IO.Path.Combine(tempfolder, "patch-" + patches.Count.ToString() + ".zip"); //Check just before we download stuff CheckLiveControl(); try { using (new Logging.Timer("Get " + bes.Key.Filename)) backend.Get(bes.Key, manifest, filename, manifest.SignatureHashes == null ? null : manifest.SignatureHashes[bes.Key.Volumenumber - 1]); } catch (BackendWrapper.HashMismathcException hme) { if (allowHashFail) { if (stat != null) stat.LogError(string.Format(Strings.Interface.FileHashFailure, hme.Message), hme); continue; } else throw; } patches.Add(new KeyValuePair<ManifestEntry,Duplicati.Library.Interface.ICompression>(be, DynamicLoader.CompressionLoader.GetModule(bes.Key.Compression, filename, m_options.RawOptions))); } } } backend.DeleteOrphans(true); return patches; }
/// <summary> /// Will attempt to read the manifest file, optionally reverting to the secondary manifest if reading one fails. /// </summary> /// <param name="backend">The backendwrapper to read from</param> /// <param name="entry">The manifest to read</param> /// <returns>The parsed manifest</returns> private Manifestfile GetManifest(BackendWrapper backend, ManifestEntry entry) { if (m_options.DontReadManifests) { Manifestfile mf = new Manifestfile(); mf.SignatureHashes = null; mf.ContentHashes = null; return mf; } if (entry.ParsedManifest != null) return entry.ParsedManifest; else if (entry.Alternate != null && entry.Alternate.ParsedManifest != null) return entry.Alternate.ParsedManifest; if (OperationProgress != null && backend.Statistics != null) OperationProgress(this, GetOperationType(), backend.Statistics.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusReadingManifest, entry.Time.ToShortDateString() + " " + entry.Time.ToShortTimeString()), ""); bool tryAlternateManifest = false; //This method has some very special logic to ensure correct handling of errors //The assumption is that it is possible to determine if the error occurred due to a // transfer problem or a corrupt file. If the former happens, the operation should // be retried, and thus an exception is thrown. If the latter, the file should // be ignored and the backup file should be used. // //We detect a parsing error, either directly or indirectly through CryptographicException, // and assume that a parsing error is an indication of a broken file. //All other errors are assumed to be transfer problems, and throws exceptions. // //This holds as long as the backend always throws an exception if a partial file // was downloaded. The FTP backend may not honor this, and some webservers // may ommit the "Content-Length" header, which will cause problems. //There is a guard agains partial downloads in BackendWrapper.GetInternal() using (new Logging.Timer("Get " + entry.Filename)) using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { try { backend.Get(entry, null, tf, null); //We now have the file decrypted, if the next step fails, // its a broken xml or invalid content tryAlternateManifest = true; Manifestfile mf = new Manifestfile(tf, m_options.SkipFileHashChecks); if (string.IsNullOrEmpty(mf.SelfFilename)) mf.SelfFilename = entry.Filename; if (mf.ContentHashes != null && entry.Alternate != null) { //Special case, the manifest has not recorded all volumes, // we must see if the alternate manifest has more volumes if (entry.Volumes.Count > mf.ContentHashes.Count) { //Do not try the alternate, we just did tryAlternateManifest = false; Logging.Log.WriteMessage(string.Format(Strings.Interface.ReadingSecondaryManifestLogMessage, entry.Alternate.Filename), Duplicati.Library.Logging.LogMessageType.Information); Manifestfile amf = null; //Read the alternate file and try to differentiate between a defect file or a partial one bool defectFile = false; try { System.IO.File.Delete(tf); backend.Get(entry.Alternate, null, tf, null); } catch (System.Security.Cryptography.CryptographicException cex) { //We assume that CryptoException means partial file Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, cex), Duplicati.Library.Logging.LogMessageType.Warning); defectFile = true; } if (!defectFile) { try { amf = new Manifestfile(tf, m_options.SkipFileHashChecks); } catch (Exception ex) { //Parsing error means partial file Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, ex), Duplicati.Library.Logging.LogMessageType.Warning); defectFile = true; } } //If the alternate manifest is correct, assign it so we have a copy if (!defectFile && amf != null) { if (string.IsNullOrEmpty(amf.SelfFilename)) amf.SelfFilename = entry.Alternate.Filename; //If the alternate manifest has more files than the primary, we use that one if (amf.ContentHashes != null && amf.ContentHashes.Count > mf.ContentHashes.Count) { entry.Alternate.ParsedManifest = amf; if (m_options.SkipFileHashChecks) { mf.SignatureHashes = null; mf.ContentHashes = null; } return amf; } } } } if (m_options.SkipFileHashChecks) { mf.SignatureHashes = null; mf.ContentHashes = null; } entry.ParsedManifest = mf; return mf; } catch (Exception ex) { //Only try secondary if the parsing/decrypting fails, not if the transfer fails if (entry.Alternate != null && (ex is System.Security.Cryptography.CryptographicException || tryAlternateManifest)) { //TODO: If it is a version error, there is no need to read the alternate version Logging.Log.WriteMessage(string.Format(Strings.Interface.PrimaryManifestReadErrorLogMessage, entry.Filename, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning); try { Logging.Log.WriteMessage(string.Format(Strings.Interface.ReadingSecondaryManifestLogMessage, entry.Alternate.Filename), Duplicati.Library.Logging.LogMessageType.Information); return GetManifest(backend, entry.Alternate); } catch (Exception ex2) { Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, ex2.Message), Duplicati.Library.Logging.LogMessageType.Warning); } } //Report the original error throw; } } }
public List<KeyValuePair<BackupEntryBase, Exception>> VerifyBackupChain() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.Verify); SetupCommonOptions(stats); List<KeyValuePair<BackupEntryBase, Exception>> results = new List<KeyValuePair<BackupEntryBase, Exception>>(); if (m_options.DontReadManifests) throw new InvalidOperationException(Strings.Interface.ManifestsMustBeRead); if (m_options.SkipFileHashChecks) throw new InvalidOperationException(Strings.Interface.CannotVerifyWithoutHashes); if (!string.IsNullOrEmpty(m_options.SignatureCachePath)) { stats.LogWarning(Strings.Interface.DisablingSignatureCacheForVerification, null); m_options.SignatureCachePath = null; } using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { //Find the spot in the chain where we start ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime); //Get the list of manifests to validate List<ManifestEntry> entries = new List<ManifestEntry>(); entries.Add(bestFit); entries.AddRange(bestFit.Incrementals); entries.Reverse(); foreach (ManifestEntry me in entries) { Manifestfile mf = null; try { mf = GetManifest(backend, me); VerifyBackupChainWithFiles(backend, me); if (mf.SignatureHashes.Count != me.Volumes.Count) results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, new Exception(string.Format(Strings.Interface.ManifestAndFileCountMismatchError, mf.SelfFilename, mf.SignatureHashes.Count, me.Volumes.Count)))); else results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, null)); } catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, ex)); } if (mf != null) { int volumes = Math.Min(mf.SignatureHashes.Count, me.Volumes.Count); for(int i = 0; i <volumes; i++) { if (m_options.Verificationlevel == VerificationLevel.Signature || m_options.Verificationlevel == VerificationLevel.Full) { try { using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) backend.Get(me.Volumes[i].Key, mf, tf, mf.SignatureHashes[i]); results.Add(new KeyValuePair<BackupEntryBase, Exception>(me.Volumes[i].Key, null)); } catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me.Volumes[i].Key, ex)); } } if (m_options.Verificationlevel == VerificationLevel.Full) { try { using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) backend.Get(me.Volumes[i].Value, mf, tf, mf.ContentHashes[i]); results.Add(new KeyValuePair<BackupEntryBase, Exception>(me.Volumes[i].Value, null)); } catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me.Volumes[i].Value, ex)); } } } } } //Re-generate verification file if (m_options.CreateVerificationFile) { //Stop any async operations if (m_options.AsynchronousUpload) backend.ExtractPendingUploads(); VerificationFile vf = new VerificationFile(entries, new FilenameStrategy(m_options)); using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { vf.Save(tf); tf.Protected = true; backend.Put(new VerificationEntry(entries[entries.Count - 1].Time), tf); } } } return results; }
private void CreateFolder() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.CreateFolder); SetupCommonOptions(stats); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) backend.CreateFolder(); }
public string Backup(string[] sources) { BackupStatistics bs = new BackupStatistics(DuplicatiOperationMode.Backup); SetupCommonOptions(bs); BackendWrapper backend = null; VerificationFile verification = null; if (m_options.DontReadManifests) throw new Exception(Strings.Interface.ManifestsMustBeReadOnBackups); if (m_options.SkipFileHashChecks) throw new Exception(Strings.Interface.CannotSkipHashChecksOnBackup); if (sources == null || sources.Length == 0) throw new Exception(Strings.Interface.NoSourceFoldersError); //Make sure they all have the same format and exist for (int i = 0; i < sources.Length; i++) { sources[i] = Utility.Utility.AppendDirSeparator(System.IO.Path.GetFullPath(sources[i])); if (!System.IO.Directory.Exists(sources[i])) throw new System.IO.IOException(String.Format(Strings.Interface.SourceFolderIsMissingError, sources[i])); } //Sanity check for duplicate folders and multiple inclusions of the same folder for (int i = 0; i < sources.Length - 1; i++) { for (int j = i + 1; j < sources.Length; j++) if (sources[i].Equals(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) throw new Exception(string.Format(Strings.Interface.SourceDirIsIncludedMultipleTimesError, sources[i])); else if (sources[i].StartsWith(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) throw new Exception(string.Format(Strings.Interface.SourceDirsAreRelatedError, sources[i], sources[j])); } if (m_options.AsynchronousUpload) { m_asyncReserved = ASYNC_RESERVED; m_allowUploadProgress = false; } //Unused, but triggers errors in the encryption setup here Library.Interface.IEncryption encryptionModule = m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); using (new Logging.Timer("Backup from " + string.Join(";", sources) + " to " + m_backend)) { try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, ""); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, ""); CheckLiveControl(); bool full = m_options.Full; if (full) bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseFlagWasSet, "full")); backend = new BackendWrapper(bs, m_backend, m_options); backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent); backend.AsyncItemProcessedEvent += new EventHandler(backend_AsyncItemProcessedEvent); m_progress = 0.0; OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); CheckLiveControl(); List<ManifestEntry> backupsets; if (full) { //This will create the target folder backend.List(false); backupsets = new List<ManifestEntry>(); } else { //This will list all files on the backend and create the target folder backupsets = backend.GetBackupSets(); } if (backupsets.Count == 0) { if (!full) bs.SetTypeReason(Strings.Interface.FullBecauseBackendIsEmpty); full = true; } else { //A prioir backup exists, extract the compression and encryption modules used in the most recent entry string compression = null; string encryption = null; for (int i = backupsets.Count - 1; compression == null && i >= 0; i--) { for (int j = backupsets[i].Incrementals.Count - 1; compression == null && j >= 0; j--) for (int k = backupsets[i].Incrementals[j].Volumes.Count - 1; compression == null && k >= 0; k--) { compression = backupsets[i].Incrementals[j].Volumes[k].Key.Compression; encryption = backupsets[i].Incrementals[j].Volumes[k].Key.EncryptionMode; if (compression != null) break; } for (int k = backupsets[i].Volumes.Count - 1; compression == null && k >= 0; k--) { compression = backupsets[i].Volumes[k].Key.Compression; encryption = backupsets[i].Volumes[k].Key.EncryptionMode; if (compression != null) break; } } if (compression != null) { m_options.SetEncryptionModuleDefault(encryption); m_options.SetCompressionModuleDefault(compression); } } string fullCriteria1 = null; string fullCriteria2 = null; if (!full) { full = DateTime.Now > m_options.FullIfOlderThan(backupsets[backupsets.Count - 1].Time); if (full) bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue)); else if (!string.IsNullOrEmpty(m_options.FullIfOlderThanValue)) fullCriteria1 = string.Format(Strings.Interface.IncrementalBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue); } if (!full && m_options.FullIfMoreThanNIncrementals > 0) { full = backupsets[backupsets.Count - 1].Incrementals.Count >= m_options.FullIfMoreThanNIncrementals; if (full) bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals)); else fullCriteria2 = string.Format(Strings.Interface.IncrementalBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals); } bs.Full = full; if (!full) { if (fullCriteria1 == null && fullCriteria2 == null) bs.SetTypeReason(Strings.Interface.IncrementalBecauseNoFlagsWereSet); else if (fullCriteria2 == null) bs.SetTypeReason(fullCriteria1); else if (fullCriteria1 == null) bs.SetTypeReason(fullCriteria2); else bs.SetTypeReason(fullCriteria1 + ". " + fullCriteria2); } List<string> controlfiles = new List<string>(); if (!string.IsNullOrEmpty(m_options.SignatureControlFiles)) controlfiles.AddRange(m_options.SignatureControlFiles.Split(System.IO.Path.PathSeparator)); int vol = 0; long totalsize = 0; Manifestfile manifest = new Manifestfile(); using (Utility.TempFolder tempfolder = new Duplicati.Library.Utility.TempFolder()) { List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = new List<KeyValuePair<ManifestEntry, Duplicati.Library.Interface.ICompression>>(); if (!full) { m_incrementalFraction = INCREMENAL_COST; List<ManifestEntry> entries = new List<ManifestEntry>(); entries.Add(backupsets[backupsets.Count - 1]); entries.AddRange(backupsets[backupsets.Count - 1].Incrementals); //Check before we start the download CheckLiveControl(); VerifyBackupChainWithFiles(backend, entries[entries.Count - 1]); if (m_options.CreateVerificationFile) verification = new VerificationFile(entries, backend.FilenameStrategy); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); patches = FindPatches(backend, entries, tempfolder, false, bs); //Check before we start the download CheckLiveControl(); Manifestfile latest = GetManifest(backend, backupsets[backupsets.Count - 1]); //Manifest version 1 does not support multiple folders if (latest.Version == 1) latest.SourceDirs = new string[] { sources[0] }; if (latest.SourceDirs.Length != sources.Length) { if (m_options.FullIfSourceFolderChanged) { Logging.Log.WriteMessage("Source folder count changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information); if (!full) bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged); full = true; } else throw new Exception(string.Format(Strings.Interface.NumberOfSourceFoldersHasChangedError, latest.SourceDirs.Length, sources.Length)); } else { if (!m_options.AllowSourceFolderChange) { foreach (string s1 in latest.SourceDirs) { bool found = false; foreach (string s2 in sources) if (s1.Equals(s2, Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) { found = true; break; } if (!found) { if (m_options.FullIfSourceFolderChanged) { Logging.Log.WriteMessage("Source folders changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information); if (!full) bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged); full = true; break; //Exit the folder loop } else throw new Exception(string.Format(Strings.Interface.SourceFoldersHasChangedError, s1)); } } manifest.SourceDirs = latest.SourceDirs; } else { manifest.SourceDirs = sources; } } } DateTime backuptime = DateTime.Now; DateTime backupchaintime; if (full) { patches.Clear(); m_incrementalFraction = 0.0; manifest.SourceDirs = sources; if (m_options.CreateVerificationFile) verification = new VerificationFile(new ManifestEntry[0], backend.FilenameStrategy); backupchaintime = backuptime; } else { backupchaintime = patches[0].Key.Time; manifest.PreviousManifestFilename = patches[patches.Count - 1].Key.Filename; manifest.PreviousManifestHash = patches[patches.Count - 1].Key.RemoteHash; } OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusBuildingFilelist, ""); bool completedWithoutChanges; using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(manifest.SourceDirs, bs, m_options.Filter, patches)) { CheckLiveControl(); dir.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupRSyncDir_ProgressEvent); dir.DisableFiletimeCheck = m_options.DisableFiletimeCheck; dir.MaxFileSize = m_options.SkipFilesLargerThan; using (new Logging.Timer("Initiating multipass")) dir.InitiateMultiPassDiff(full, m_options); string tempVolumeFolder = m_options.AsynchronousUpload ? m_options.AsynchronousUploadFolder : m_options.TempDir; bool done = false; while (!done && totalsize < m_options.MaxSize) { using (new Logging.Timer("Multipass " + (vol + 1).ToString())) using (Utility.TempFile signaturefile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString()))) using (Utility.TempFile contentfile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString()))) { OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusCreatingVolume, vol + 1), ""); CheckLiveControl(); using (Library.Interface.ICompression signaturearchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, signaturefile, m_options.RawOptions)) using (Library.Interface.ICompression contentarchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, contentfile, m_options.RawOptions)) { //If we are all out, stop now, this may cause incomplete partial files if (m_options.MaxSize - totalsize < (contentarchive.FlushBufferSize + backend.FileSizeOverhead)) break; //Add signature files to archive foreach (string s in controlfiles) if (!string.IsNullOrEmpty(s)) using (System.IO.Stream cs = signaturearchive.CreateFile(System.IO.Path.Combine(RSync.RSyncDir.CONTROL_ROOT, System.IO.Path.GetFileName(s)))) using (System.IO.FileStream fs = System.IO.File.OpenRead(s)) Utility.Utility.CopyStream(fs, cs); //Only add control files to the very first volume controlfiles.Clear(); done = dir.MakeMultiPassDiff(signaturearchive, contentarchive, (Math.Min(m_options.VolumeSize, m_options.MaxSize - totalsize)) - backend.FileSizeOverhead); //TODO: This is not the correct size, we need to account for file size overhead as well totalsize += signaturearchive.Size; totalsize += contentarchive.Size; //TODO: This is not the best way to determine this if (totalsize >= m_options.MaxSize) dir.FinalizeMultiPass(signaturearchive, contentarchive, long.MaxValue); } completedWithoutChanges = done && !dir.AnyChangesFound; if (m_options.UploadUnchangedBackups || full) completedWithoutChanges = false; if (!completedWithoutChanges) { if (m_options.AsynchronousUpload) { m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload; m_allowUploadProgress = true; m_allowUploadProgressAfter = DateTime.Now.AddSeconds(1); } else OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingContentVolume, vol + 1), ""); //Last check before we upload, we do not interrupt transfers CheckLiveControl(); //The backendwrapper will remove these signaturefile.Protected = true; contentfile.Protected = true; ContentEntry ce = new ContentEntry(backuptime, full, vol + 1); SignatureEntry se = new SignatureEntry(backuptime, full, vol + 1); using (new Logging.Timer("Writing delta file " + (vol + 1).ToString())) backend.Put(ce, contentfile); if (!m_options.AsynchronousUpload) OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingSignatureVolume, vol + 1), ""); using (new Logging.Timer("Writing remote signatures")) backend.Put(se, signaturefile); manifest.AddEntries(ce, se); if (verification != null) { verification.AddFile(ce); verification.AddFile(se); } } } if (!completedWithoutChanges) { //The backend wrapper will remove these Utility.TempFile mf = new Duplicati.Library.Utility.TempFile(); using (new Logging.Timer("Writing manifest " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK"))) { //Alternate primary/secondary ManifestEntry mfe = new ManifestEntry(backuptime, full, manifest.SignatureHashes.Count % 2 != 0); manifest.SelfFilename = backend.GenerateFilename(mfe); manifest.Save(mf); if (!m_options.AsynchronousUpload) OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingManifestVolume, vol + 1), ""); //Write the file mf.Protected = true; backend.Put(mfe, mf); if (verification != null) verification.UpdateManifest(mfe); } if (verification != null) { using (new Logging.Timer("Writing verification " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK"))) { Utility.TempFile vt = new Duplicati.Library.Utility.TempFile(); verification.Save(vt); if (!m_options.AsynchronousUpload) OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusUploadingVerificationVolume, ""); vt.Protected = true; backend.Put(new VerificationEntry(backupchaintime), vt); } } if (m_options.AsynchronousUpload) m_allowUploadProgress = false; //The file volume counter vol++; } } } //If we are running asynchronous, we now enter the end-game if (m_options.AsynchronousUpload) { m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload; m_allowUploadProgress = true; m_allowUploadProgressAfter = DateTime.Now; //Before we clear the temp folder, we need to ensure that all volumes are uploaded. //To allow the UI to show some progress while uploading, we perform the remaining // uploads synchronous List<KeyValuePair<BackupEntryBase, string>> pendingUploads = backend.ExtractPendingUploads(); //Figure out what volume number we are at foreach (KeyValuePair<BackupEntryBase, string> p in pendingUploads) if (p.Key is ManifestEntry) vol--; double unitcost = m_asyncReserved / pendingUploads.Count; //The upload each remaining volume in order foreach (KeyValuePair<BackupEntryBase, string> p in pendingUploads) { string msg; if (p.Key is ManifestEntry) { vol++; msg = string.Format(Strings.Interface.StatusUploadingManifestVolume, vol); } else if (p.Key is SignatureEntry) msg = string.Format(Strings.Interface.StatusUploadingSignatureVolume, ((SignatureEntry)p.Key).Volumenumber); else if (p.Key is ContentEntry) { msg = string.Format(Strings.Interface.StatusUploadingContentVolume, ((ContentEntry)p.Key).Volumenumber); //We allow a stop or pause request here CheckLiveControl(); } else if (p.Key is VerificationEntry) msg = Strings.Interface.StatusUploadingVerificationVolume; else throw new InvalidOperationException(); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, msg, ""); backend.Put(p.Key, p.Value); m_asyncReserved -= unitcost; m_progress += unitcost; } } } } catch(Exception ex) { //If this is a controlled user-requested stop, wait for the current upload to complete if (backend != null && ex is LiveControl.ExecutionStoppedException) { try { if (m_options.AsynchronousUpload) { m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload; m_allowUploadProgress = true; m_allowUploadProgressAfter = DateTime.Now; //Wait for the current upload to complete and then delete all remaining temporary files foreach (KeyValuePair<BackupEntryBase, string> p in backend.ExtractPendingUploads()) try { if (System.IO.File.Exists(p.Value)) System.IO.File.Delete(p.Value); } catch { } //Better to delete as many as possible rather than choke on a single file } } catch { } //We already have an exception, just go with that } if (backend == null || backend.ManifestUploads == 0) { Logging.Log.WriteMessage(string.Format(Strings.Interface.ErrorRunningBackup, ex.Message), Logging.LogMessageType.Error); throw; //This also activates "finally", unlike in other languages... } Logging.Log.WriteMessage(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), Logging.LogMessageType.Warning); bs.LogError(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), ex); } finally { m_progress = 100.0; if (backend != null) try { backend.Dispose(); } catch { } if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } } bs.EndTime = DateTime.Now; return bs.ToString(); }
public string DeleteAllButNFull() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButNFull); SetupCommonOptions(stats); int x = Math.Max(0, m_options.DeleteAllButNFull); StringBuilder sb = new StringBuilder(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List<ManifestEntry> entries = backend.GetBackupSets(); List<ManifestEntry> toremove = new List<ManifestEntry>(); while (entries.Count > x) { if (entries.Count == 1 && !m_options.AllowFullRemoval) { sb.AppendLine(string.Format(Strings.Interface.NotDeletingLastFullMessage, entries[0].Time)); break; } ManifestEntry be = entries[0]; entries.RemoveAt(0); be.Incrementals.Reverse(); toremove.AddRange(be.Incrementals); toremove.Add(be); } if (entries.Count == 0 && toremove.Count > 0 && !m_options.AllowFullRemoval) throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); } finally { if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } return sb.ToString(); }
public IList<string> ListCurrentFiles() { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.ListCurrentFiles); SetupCommonOptions(rs); Utility.FilenameFilter filter = m_options.Filter; DateTime timelimit = m_options.RestoreTime; if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.List, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List<string> res; using (BackendWrapper backend = new BackendWrapper(rs, m_backend, m_options)) using (Utility.TempFolder basefolder = new Duplicati.Library.Utility.TempFolder()) { ManifestEntry bestFit = backend.GetBackupSet(timelimit); List<ManifestEntry> entries = new List<ManifestEntry>(); entries.Add(bestFit); entries.AddRange(bestFit.Incrementals); List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = FindPatches(backend, entries, basefolder, false, rs); using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(new string[] { basefolder }, rs, filter, patches)) res = dir.UnmatchedFiles(); } if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.List, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); return res; }
public string[] ListSourceFolders() { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.ListSourceFolders); SetupCommonOptions(rs); if (m_options.DontReadManifests) throw new Exception(Strings.Interface.ManifestsMustBeRead); DateTime timelimit = m_options.RestoreTime; if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.List, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); string[] res; using (BackendWrapper backend = new BackendWrapper(rs, m_backend, m_options)) using (Utility.TempFile mfile = new Duplicati.Library.Utility.TempFile()) { ManifestEntry bestFit = backend.GetBackupSet(timelimit); backend.Get(bestFit, null, mfile, null); res = new Manifestfile(mfile, m_options.SkipFileHashChecks).SourceDirs; } if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.List, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); return res; }
public string[] List() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.List); SetupCommonOptions(stats); List<string> res = new List<string>(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.List, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); foreach (Duplicati.Library.Interface.IFileEntry fe in backend.List(false)) res.Add(fe.Name); if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.List, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); return res.ToArray(); } }
public List<KeyValuePair<RSync.RSyncDir.PatchFileType, string>> ListActualSignatureFiles() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.ListActualSignatureFiles); SetupCommonOptions(stats); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime); if (bestFit.Incrementals.Count > 0) //Get the most recent incremental bestFit = bestFit.Incrementals[bestFit.Incrementals.Count - 1]; using (Utility.TempFolder folder = new Duplicati.Library.Utility.TempFolder()) { List<Library.Interface.ICompression> patches = new List<Duplicati.Library.Interface.ICompression>(); foreach (KeyValuePair<ManifestEntry, Library.Interface.ICompression> entry in FindPatches(backend, new List<ManifestEntry>(new ManifestEntry[] { bestFit }), folder, false, stats)) patches.Add(entry.Value); using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(new string[] { folder }, stats, null)) return dir.ListPatchFiles(patches); } } }
public List<ManifestEntry> GetBackupSets() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.GetBackupSets); SetupCommonOptions(stats); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) return backend.GetBackupSets(); }
/// <summary> /// Reads through a backup and finds the last backup entry that has a specific file /// </summary> /// <returns></returns> public List<KeyValuePair<string, DateTime>> FindLastFileVersion() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.FindLastFileVersion); SetupCommonOptions(stats); if (m_options.DontReadManifests) throw new Exception(Strings.Interface.ManifestsMustBeRead); if (string.IsNullOrEmpty(m_options.FileToRestore)) throw new Exception(Strings.Interface.NoFilesGivenError); string[] filesToFind = m_options.FileToRestore.Split(System.IO.Path.PathSeparator); KeyValuePair<string, DateTime>[] results = new KeyValuePair<string, DateTime>[filesToFind.Length]; for (int i = 0; i < results.Length; i++) results[i] = new KeyValuePair<string, DateTime>(filesToFind[i], new DateTime(0)); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { //Extract the full backup set list List<ManifestEntry> fulls = backend.GetBackupSets(); //Flatten the list List<ManifestEntry> workList = new List<ManifestEntry>(); //The list is oldest first, this function work newest first fulls.Reverse(); foreach (ManifestEntry f in fulls) { f.Incrementals.Reverse(); workList.AddRange(f.Incrementals); workList.Add(f); } bool warned_manifest_v1 = false; foreach (ManifestEntry mf in workList) { List<Manifestfile.HashEntry> signatureHashes = null; Manifestfile mfi; using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { backend.Get(mf, null, tf, null); mfi = new Manifestfile(tf, m_options.SkipFileHashChecks); if (!m_options.SkipFileHashChecks) signatureHashes = mfi.SignatureHashes; } //If there are no volumes, don't stop here bool any_unmatched = true; if (stats != null && !warned_manifest_v1 && (mfi.SourceDirs == null || mfi.SourceDirs.Length == 0)) { warned_manifest_v1 = true; stats.LogWarning(Strings.Interface.ManifestVersionRequiresRelativeNamesWarning, null); } foreach(KeyValuePair<SignatureEntry, ContentEntry> e in mf.Volumes) using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { //Skip non-approved signature files if (signatureHashes != null && e.Key.Volumenumber > signatureHashes.Count) { stats.LogWarning(string.Format(Strings.Interface.SkippedUnlistedSignatureFileWarning, e.Key.Filename), null); continue; } backend.Get(e.Key, mfi, tf, signatureHashes == null ? null : signatureHashes[e.Key.Volumenumber - 1]); any_unmatched = false; RSync.RSyncDir.ContainsFile(mfi, filesToFind, DynamicLoader.CompressionLoader.GetModule(e.Key.Compression, tf, m_options.RawOptions)); for (int i = 0; i < filesToFind.Length; i++) { if (results[i].Value.Ticks == 0 && string.IsNullOrEmpty(filesToFind[i])) results[i] = new KeyValuePair<string,DateTime>(results[i].Key, mf.Time); else any_unmatched = true; } if (!any_unmatched) break; } if (!any_unmatched) break; } return new List<KeyValuePair<string,DateTime>>(results); } }
public string DeleteOlderThan() { StringBuilder sb = new StringBuilder(); CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteOlderThan); SetupCommonOptions(stats); DateTime expires = m_options.RemoveOlderThan; using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List<ManifestEntry> entries = backend.GetBackupSets(); List<ManifestEntry> toremove = new List<ManifestEntry>(); while (entries.Count > 0 && entries[0].Time <= expires) { if (entries.Count == 1 && !m_options.AllowFullRemoval) { sb.AppendLine(string.Format(Strings.Interface.NotDeletingLastFullMessage, entries[0].Time)); break; } ManifestEntry be = entries[0]; entries.RemoveAt(0); bool hasNewer = false; foreach (ManifestEntry bex in be.Incrementals) if (bex.Time >= expires) { hasNewer = true; break; } if (hasNewer) { List<ManifestEntry> t = new List<ManifestEntry>(be.Incrementals); t.Insert(0, be); for (int i = 0; i < t.Count; i++) if (t[i].Time <= expires) sb.AppendLine(string.Format(Strings.Interface.NotDeletingBackupSetMessage, t[i].Time.ToString(System.Globalization.CultureInfo.InvariantCulture))); break; } else { be.Incrementals.Reverse(); toremove.AddRange(be.Incrementals); toremove.Add(be); } } if (entries.Count == 0 && toremove.Count > 0 && !m_options.AllowFullRemoval) throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); } finally { if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } return sb.ToString(); }
/// <summary> /// Verifies the backup chain for producing a new backup on top. /// This will check that all files are accounted for in the file list. /// </summary> /// <param name="entry">The newest entry to check</param> private void VerifyBackupChainWithFiles(BackendWrapper backend, ManifestEntry entry) { VerifyManifestChain(backend, entry); string errorMessage = Environment.NewLine + Strings.Interface.DeleteManifestsSuggestion + Environment.NewLine + Environment.NewLine; while (entry != null) { Manifestfile parsed = GetManifest(backend, entry); errorMessage += entry.Filename + Environment.NewLine; if (entry.Volumes.Count != parsed.SignatureHashes.Count || entry.Volumes.Count != parsed.ContentHashes.Count) { //If we have an extra set, the connection could have died right before the manifest was uploaded if (parsed.SignatureHashes.Count == parsed.ContentHashes.Count && entry.Volumes.Count - 1 == parsed.ContentHashes.Count) { backend.AddOrphan(entry.Volumes[entry.Volumes.Count - 1].Value); backend.AddOrphan(entry.Volumes[entry.Volumes.Count - 1].Key); entry.Volumes.RemoveAt(entry.Volumes.Count - 1); } else { throw new Exception( string.Format(Strings.Interface.ManifestAndFileCountMismatchError, entry.Filename, parsed.SignatureHashes.Count, entry.Volumes.Count) + errorMessage ); } } for(int i = 0; i < entry.Volumes.Count; i++) { if (entry.Volumes[i].Key.Filesize > 0 && parsed.SignatureHashes[i].Size > 0 && entry.Volumes[i].Key.Filesize != parsed.SignatureHashes[i].Size) throw new Exception( string.Format(Strings.Interface.FileSizeMismatchError, entry.Volumes[i].Key.Filename, entry.Volumes[i].Key.Filesize, parsed.SignatureHashes[i].Size) + errorMessage ); if (entry.Volumes[i].Value.Filesize >= 0 && parsed.ContentHashes[i].Size >= 0 && entry.Volumes[i].Value.Filesize != parsed.ContentHashes[i].Size) throw new Exception( string.Format(Strings.Interface.FileSizeMismatchError, entry.Volumes[i].Value.Filename, entry.Volumes[i].Value.Filesize, parsed.ContentHashes[i].Size) + errorMessage ); if (!string.IsNullOrEmpty(parsed.SignatureHashes[i].Name) && !parsed.SignatureHashes[i].Name.Equals(entry.Volumes[i].Key.Fileentry.Name, StringComparison.InvariantCultureIgnoreCase)) throw new Exception( string.Format(Strings.Interface.FilenameMismatchError, parsed.SignatureHashes[i].Name, entry.Volumes[i].Key.Fileentry.Name) + errorMessage ); if (!string.IsNullOrEmpty(parsed.ContentHashes[i].Name) && !parsed.ContentHashes[i].Name.Equals(entry.Volumes[i].Value.Fileentry.Name, StringComparison.InvariantCultureIgnoreCase)) throw new Exception( string.Format(Strings.Interface.FilenameMismatchError, parsed.ContentHashes[i].Name, entry.Volumes[i].Value.Fileentry.Name) + errorMessage ); } entry = entry.Previous; } }
public string Restore(string[] target) { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.Restore); SetupCommonOptions(rs); m_progress = 0; BackendWrapper backend = null; m_restorePatches = 0; using (new Logging.Timer("Restore from " + m_backend + " to " + string.Join(System.IO.Path.PathSeparator.ToString(), target))) { try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Restore, rs.OperationMode, -1, -1, Strings.Interface.StatusStarted, ""); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, -1, -1, Strings.Interface.StatusStarted, ""); Utility.FilenameFilter filter = m_options.Filter; //Filter is prefered, if both file and filter is specified if (!m_options.HasFilter && !string.IsNullOrEmpty(m_options.FileToRestore)) { List<Utility.IFilenameFilter> list = new List<Duplicati.Library.Utility.IFilenameFilter>(); list.Add(new Utility.FilelistFilter(true, m_options.FileToRestore.Split(System.IO.Path.PathSeparator))); list.Add(new Utility.RegularExpressionFilter(false, ".*")); filter = new Duplicati.Library.Utility.FilenameFilter(list); } backend = new BackendWrapper(rs, m_backend, m_options); backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime); //We will need all the manifests downloaded anyway if (!m_options.DontReadManifests) { if (bestFit.Incrementals.Count > 0) VerifyManifestChain(backend, bestFit.Incrementals[bestFit.Incrementals.Count - 1]); else VerifyManifestChain(backend, bestFit); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); } m_progress = INCREMENAL_COST; List<ManifestEntry> entries = new List<ManifestEntry>(); entries.Add(bestFit); entries.AddRange(bestFit.Incrementals); int patchno = 0; foreach (ManifestEntry be in entries) m_restorePatches += be.Volumes.Count; Manifestfile rootManifest = GetManifest(backend, bestFit); int sourceDirCount = (rootManifest.SourceDirs == null || rootManifest.SourceDirs.Length == 0) ? 1 : rootManifest.SourceDirs.Length; //After reading the first manifest, we know the source folder count if ((rootManifest.SourceDirs == null || rootManifest.SourceDirs.Length == 0) && target.Length > 1) { //V1 support rs.LogWarning(string.Format(Strings.Interface.TooManyTargetFoldersWarning, 1, target.Length), null); Array.Resize(ref target, 1); } else if (target.Length > sourceDirCount) { //If we get too many, we can just cut them off rs.LogWarning(string.Format(Strings.Interface.TooManyTargetFoldersWarning, sourceDirCount, target.Length), null); Array.Resize(ref target, rootManifest.SourceDirs.Length); } else if (target.Length != 1 && target.Length < sourceDirCount) { //If we get too few, we have to bail throw new Exception(string.Format(Strings.Interface.TooFewTargetFoldersError, sourceDirCount, target.Length)); } else if (target.Length == 1 && sourceDirCount > 1) { //If there is just one target folder, we automatically compose target subfolders string[] newtargets = new string[rootManifest.SourceDirs.Length]; List<string> suggestions = new List<string>(); for (int i = 0; i < rootManifest.SourceDirs.Length; i++) { string s = rootManifest.SourceDirs[i]; //HACK: We use a leading / in the path name to detect source OS // all paths are absolute, so this detects all unix like systems string dirSepChar = s.StartsWith("/") ? "/" : "\\"; if (s.EndsWith(dirSepChar)) s = s.Substring(0, s.Length - 1); int lix = s.LastIndexOf(dirSepChar); if (lix < 0 || lix + 1 >= s.Length) s = i.ToString(); else s = s.Substring(lix + 1); foreach (char c in System.IO.Path.GetInvalidFileNameChars()) s = s.Replace(c, '_'); suggestions.Add(s); } Dictionary<string, int> duplicates = new Dictionary<string, int>(Library.Utility.Utility.ClientFilenameStringComparer); for (int i = 0; i < suggestions.Count; i++) if (duplicates.ContainsKey(suggestions[i])) duplicates[suggestions[i]]++; else duplicates[suggestions[i]] = 1; for (int i = 0; i < newtargets.Length; i++) { string suffix = duplicates[suggestions[i]] > 1 ? i.ToString() : suggestions[i]; newtargets[i] = System.IO.Path.Combine(target[0], suffix); } target = newtargets; } //Make sure all targets exist foreach(string s in target) if (!System.IO.Directory.Exists(s)) System.IO.Directory.CreateDirectory(s); using (RSync.RSyncDir sync = new Duplicati.Library.Main.RSync.RSyncDir(target, rs, filter)) { sync.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(RestoreRSyncDir_ProgressEvent); foreach (ManifestEntry be in entries) { m_progress = ((1.0 - INCREMENAL_COST) * (patchno / (double)m_restorePatches)) + INCREMENAL_COST; CheckLiveControl(); Manifestfile manifest = be == bestFit ? rootManifest : GetManifest(backend, be); CheckLiveControl(); foreach (KeyValuePair<SignatureEntry, ContentEntry> vol in be.Volumes) { ContentEntry contentVol = vol.Value; SignatureEntry signatureVol = vol.Key; m_progress = ((1.0 - INCREMENAL_COST) * (patchno / (double)m_restorePatches)) + INCREMENAL_COST; //Skip nonlisted if (manifest.ContentHashes != null && contentVol.Volumenumber > manifest.ContentHashes.Count) { Logging.Log.WriteMessage(string.Format(Strings.Interface.SkippedContentVolumeLogMessage, contentVol.Volumenumber), Duplicati.Library.Logging.LogMessageType.Warning); rs.LogWarning(string.Format(Strings.Interface.SkippedContentVolumeLogMessage, contentVol.Volumenumber), null); patchno++; continue; } using (Utility.TempFile patchzip = new Duplicati.Library.Utility.TempFile()) { OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusPatching, patchno + 1), ""); CheckLiveControl(); if (m_options.HasFilter || !string.IsNullOrEmpty(m_options.FileToRestore)) { bool hasFiles = false; using (Utility.TempFile sigFile = new Duplicati.Library.Utility.TempFile()) { OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusDownloadingSignatureVolume, patchno + 1), ""); try { using (new Logging.Timer("Get " + signatureVol.Filename)) backend.Get(signatureVol, manifest, sigFile, manifest.SignatureHashes == null ? null : manifest.SignatureHashes[signatureVol.Volumenumber - 1]); } catch (BackendWrapper.HashMismathcException hme) { hasFiles = true; rs.LogError(string.Format(Strings.Interface.FileHashFailure, hme.Message), hme); } if (!hasFiles) using (Library.Interface.ICompression patch = DynamicLoader.CompressionLoader.GetModule(signatureVol.Compression, sigFile, m_options.RawOptions)) { foreach(KeyValuePair<RSync.RSyncDir.PatchFileType, string> k in sync.ListPatchFiles(patch)) if (filter.ShouldInclude("", System.IO.Path.DirectorySeparatorChar.ToString() + k.Value)) { //TODO: Perhaps a bit much to download the content archive // if the file is only marked for deletion? hasFiles = true; break; } } } if (!hasFiles) { //Avoid downloading the content file patchno++; continue; } } OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusDownloadingContentVolume, patchno + 1), ""); using (new Logging.Timer("Get " + contentVol.Filename)) backend.Get(contentVol, manifest, patchzip, manifest.ContentHashes == null ? null : manifest.ContentHashes[contentVol.Volumenumber - 1]); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusPatching, patchno + 1), ""); using (new Logging.Timer((patchno == 0 ? "Full restore to: " : "Incremental restore " + patchno.ToString() + " to: ") + string.Join(System.IO.Path.PathSeparator.ToString(), target))) using (Library.Interface.ICompression patch = DynamicLoader.CompressionLoader.GetModule(contentVol.Compression, patchzip, m_options.RawOptions)) sync.Patch(target, patch); } patchno++; } //Make sure there are no partial files, as partial files are not allowed to span backup sets sync.FinalizeRestore(); } } } finally { if (backend != null) backend.Dispose(); if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } } rs.EndTime = DateTime.Now; return rs.ToString(); }
/// <summary> /// Validates the manifest chain, starting by evaluating the current manifest and going backwards in the chain /// </summary> /// <param name="backend">The backend wrapper</param> /// <param name="entry">The entry to verify</param> private void VerifyManifestChain(BackendWrapper backend, ManifestEntry entry) { if (m_options.DontReadManifests) throw new InvalidOperationException(Strings.Interface.CannotVerifyChain); while (entry.Previous != null) { Manifestfile parsed = GetManifest(backend, entry); //If this manifest is not version 3, the chain verification stops here if (parsed.Version < 3) return; ManifestEntry previous = entry.Previous; if (entry.Previous.Alternate != null && entry.Previous.Alternate.Filename == parsed.PreviousManifestFilename) previous = entry.Previous.Alternate; if (parsed.PreviousManifestFilename != previous.Filename) throw new System.IO.InvalidDataException(string.Format(Strings.Interface.PreviousManifestFilenameMismatchError, entry.Filename, parsed.PreviousManifestFilename, previous.Filename)); if (!m_options.SkipFileHashChecks) { //Load the Remotehash property GetManifest(backend, previous); if (parsed.PreviousManifestHash != previous.RemoteHash) throw new System.IO.InvalidDataException(string.Format(Strings.Interface.PreviousManifestHashMismatchError, entry.Filename, parsed.PreviousManifestHash, previous.RemoteHash)); } entry = previous; } }
/// <summary> /// Restores control files added to a backup. /// </summary> /// <param name="target">The folder into which to restore the files</param> /// <returns>A restore report</returns> public string RestoreControlFiles(string target) { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.RestoreControlfiles); SetupCommonOptions(rs); BackendWrapper backend = null; using (new Logging.Timer("Restore control files from " + m_backend + " to " + target)) { try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); backend = new BackendWrapper(rs, m_backend, m_options); List<ManifestEntry> attempts = backend.GetBackupSets(); List<ManifestEntry> flatlist = new List<ManifestEntry>(); foreach (ManifestEntry be in attempts) { flatlist.Add(be); flatlist.AddRange(be.Incrementals); } flatlist.Reverse(); string prefix = Utility.Utility.AppendDirSeparator(RSync.RSyncDir.CONTROL_ROOT); foreach (ManifestEntry be in flatlist) { if (be.Volumes.Count > 0) using(Utility.TempFile z = new Duplicati.Library.Utility.TempFile()) { OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, string.Format(Strings.Interface.StatusReadingIncrementalFile, be.Volumes[0].Key.Filename), ""); Manifestfile mf = GetManifest(backend, be); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, string.Format(Strings.Interface.StatusReadingIncrementalFile, be.Volumes[0].Key.Filename), ""); using (new Logging.Timer("Get " + be.Volumes[0].Key.Filename)) backend.Get(be.Volumes[0].Key, mf, z, mf.SignatureHashes == null ? null : mf.SignatureHashes[0]); using(Library.Interface.ICompression fz = DynamicLoader.CompressionLoader.GetModule(be.Volumes[0].Key.Compression, z, m_options.RawOptions)) { bool any = false; foreach (string f in fz.ListFiles(prefix)) { any = true; using (System.IO.Stream s1 = fz.OpenRead(f)) using (System.IO.Stream s2 = System.IO.File.Create(System.IO.Path.Combine(target, f.Substring(prefix.Length)))) Utility.Utility.CopyStream(s1, s2); } if (any) break; rs.LogError(string.Format(Strings.Interface.FailedToFindControlFilesMessage, be.Volumes[0].Key.Filename), null); } } } } finally { if (backend != null) backend.Dispose(); if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } } rs.EndTime = DateTime.Now; return rs.ToString(); }
public string Cleanup() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.CleanUp); SetupCommonOptions(stats); bool anyRemoved = false; StringBuilder sb = new StringBuilder(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { List<ManifestEntry> sorted = backend.GetBackupSets(); List<ManifestEntry> entries = new List<ManifestEntry>(); entries.AddRange(sorted); foreach (ManifestEntry be in sorted) entries.AddRange(be.Incrementals); string cleanup = backend.DeleteOrphans(false); if (!string.IsNullOrEmpty(cleanup)) sb.AppendLine(cleanup); if (m_options.SkipFileHashChecks) throw new Exception(Strings.Interface.CannotCleanWithoutHashesError); if (m_options.DontReadManifests) throw new Exception(Strings.Interface.CannotCleanWithoutHashesError); //We need the manifests anyway, so we verify the chain if (entries.Count > 0) VerifyManifestChain(backend, entries[0]); //Now compare the actual filelist with the manifest foreach (ManifestEntry be in entries) { Manifestfile manifest = GetManifest(backend, be); int count = manifest.ContentHashes.Count; for (int i = count - 1; i < be.Volumes.Count; i++) { anyRemoved = true; string sigmsg = string.Format(Strings.Interface.RemovingPartialFilesMessage, be.Volumes[i].Key.Filename); string cntmsg = string.Format(Strings.Interface.RemovingPartialFilesMessage, be.Volumes[i].Value.Filename); Logging.Log.WriteMessage(sigmsg, Duplicati.Library.Logging.LogMessageType.Information); Logging.Log.WriteMessage(cntmsg, Duplicati.Library.Logging.LogMessageType.Information); sb.AppendLine(sigmsg); sb.AppendLine(cntmsg); if (m_options.Force) { backend.Delete(be.Volumes[i].Key); backend.Delete(be.Volumes[i].Value); } } } } if (!m_options.Force && anyRemoved) { Logging.Log.WriteMessage(Strings.Interface.FilesAreNotForceDeletedMessage, Duplicati.Library.Logging.LogMessageType.Information); sb.AppendLine(Strings.Interface.FilesAreNotForceDeletedMessage); } return sb.ToString(); //TODO: Write a message here? }
public string DeleteAllButN() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButN); SetupCommonOptions(stats); int x = Math.Max(0, m_options.DeleteAllButNFull); StringBuilder sb = new StringBuilder(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List<ManifestEntry> flatlist = new List<ManifestEntry>(); List<ManifestEntry> entries = backend.GetBackupSets(); //Get all backups as a flat list foreach (ManifestEntry me in entries) { flatlist.Add(me); flatlist.AddRange(me.Incrementals); } //Now remove all but those requested List<ManifestEntry> toremove = new List<ManifestEntry>(); while (flatlist.Count > x) { toremove.Add(flatlist[0]); flatlist.RemoveAt(0); } //If there are still chains left, make sure we do not end up with a partial chain if (!m_options.AllowFullRemoval || flatlist.Count != 0) { //Go back until we have a full chain while (toremove.Count > 0 && (flatlist.Count == 0 || !flatlist[0].IsFull)) { sb.AppendLine(string.Format(Strings.Interface.NotDeletingBackupSetMessage, toremove[toremove.Count - 1].Time)); flatlist.Insert(0, toremove[toremove.Count - 1]); toremove.RemoveAt(toremove.Count - 1); } } if (toremove.Count > 0 && !m_options.AllowFullRemoval && (flatlist.Count == 0 || !flatlist[0].IsFull)) throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); } finally { if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } return sb.ToString(); }