private string RemoveBackupSets(BackendWrapper backend, List<ManifestEntry> entries)
{
StringBuilder sb = new StringBuilder();
sb.Append(backend.FinishDeleteTransaction(false));
if (entries.Count > 0)
{
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
System.Xml.XmlNode root = doc.AppendChild(doc.CreateElement("files"));
root.Attributes.Append(doc.CreateAttribute("version")).Value = "1";
foreach (ManifestEntry me in entries)
{
if (me.Alternate != null)
root.AppendChild(doc.CreateElement("file")).InnerText = me.Alternate.Filename;
root.AppendChild(doc.CreateElement("file")).InnerText = me.Filename;
if (me.Verification != null)
root.AppendChild(doc.CreateElement("file")).InnerText = me.Verification.Filename;
foreach (KeyValuePair<SignatureEntry, ContentEntry> kx in me.Volumes)
{
root.AppendChild(doc.CreateElement("file")).InnerText = kx.Key.Filename;
root.AppendChild(doc.CreateElement("file")).InnerText = kx.Value.Filename;
}
}
if (m_options.Force)
{
using (TempFile tf = new TempFile())
{
doc.Save(tf);
tf.Protected = true;
backend.WriteDeleteTransactionFile(tf);
}
}
foreach (ManifestEntry me in entries)
{
sb.AppendLine(string.Format(Strings.Interface.DeletingBackupSetMessage, me.Time.ToString(System.Globalization.CultureInfo.InvariantCulture)));
if (m_options.Force)
{
//Delete manifest
if (me.Alternate != null)
backend.Delete(me.Alternate);
backend.Delete(me);
if (me.Verification != null)
backend.Delete(me.Verification);
foreach (KeyValuePair<SignatureEntry, ContentEntry> kx in me.Volumes)
{
backend.Delete(kx.Key);
backend.Delete(kx.Value);
}
}
}
if (m_options.Force)
backend.RemoveDeleteTransactionFile();
if (!m_options.Force && entries.Count > 0)
sb.AppendLine(Strings.Interface.FilesAreNotForceDeletedMessage);
}
return sb.ToString();
}
/// <summary>
/// Downloads all required signature files from the backend.
/// </summary>
/// <param name="backend">The backend to read from</param>
/// <param name="entries">The flattened list of manifests</param>
/// <param name="tempfolder">The tempfolder set for this operation</param>
/// <param name="allowHashFail">True to ignore files with failed hash signature</param>
/// <returns>A list of file archives</returns>
private List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> FindPatches(BackendWrapper backend, List<ManifestEntry> entries, string tempfolder, bool allowHashFail, CommunicationStatistics stat)
{
List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = new List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>>();
using (new Logging.Timer("Reading incremental data"))
{
OperationProgress(this, GetOperationType(), stat.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementalData, "");
//Calculate the total number of files to download
//, and verify their order
int incCount = 0;
foreach (ManifestEntry be in entries)
{
int volNo = 0;
//Prevent order based bugs
if (entries.IndexOf(be) > 0)
if (entries[entries.IndexOf(be) - 1].Time >= be.Time)
throw new Exception(Strings.Interface.BadSortingDetectedError);
incCount++;
foreach (KeyValuePair<SignatureEntry, ContentEntry> bes in be.Volumes)
{
incCount++;
if (volNo + 1 != bes.Key.Volumenumber || bes.Key.Volumenumber != bes.Value.Volumenumber)
throw new Exception(Strings.Interface.BadVolumeSortOrder);
volNo++;
}
}
//The incremental part has a fixed cost, and each file has a fixed fraction of that
double unitCost = m_incrementalFraction / incCount;
//Ensure that the manifest chain has not been tampered with
// since we will read all manifest files anyway, there is no harm in doing it here
if (!m_options.DontReadManifests && entries.Count > 0)
VerifyManifestChain(backend, entries[entries.Count - 1]);
foreach (ManifestEntry be in entries)
{
m_progress += unitCost;
Manifestfile manifest = GetManifest(backend, be);
foreach (KeyValuePair<SignatureEntry, ContentEntry> bes in be.Volumes)
{
m_progress += unitCost;
//Skip non-listed incrementals
if (manifest.SignatureHashes != null && bes.Key.Volumenumber > manifest.SignatureHashes.Count)
{
backend.AddOrphan(bes.Key);
backend.AddOrphan(bes.Value);
continue;
}
OperationProgress(this, GetOperationType(), stat.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusReadingSignatureFile, be.Time.ToShortDateString() + " " + be.Time.ToShortTimeString(), bes.Key.Volumenumber), "");
string filename = System.IO.Path.Combine(tempfolder, "patch-" + patches.Count.ToString() + ".zip");
//Check just before we download stuff
CheckLiveControl();
try
{
using (new Logging.Timer("Get " + bes.Key.Filename))
backend.Get(bes.Key, manifest, filename, manifest.SignatureHashes == null ? null : manifest.SignatureHashes[bes.Key.Volumenumber - 1]);
}
catch (BackendWrapper.HashMismathcException hme)
{
if (allowHashFail)
{
if (stat != null)
stat.LogError(string.Format(Strings.Interface.FileHashFailure, hme.Message), hme);
continue;
}
else
throw;
}
patches.Add(new KeyValuePair<ManifestEntry,Duplicati.Library.Interface.ICompression>(be, DynamicLoader.CompressionLoader.GetModule(bes.Key.Compression, filename, m_options.RawOptions)));
}
}
}
backend.DeleteOrphans(true);
return patches;
}
/// <summary>
/// Will attempt to read the manifest file, optionally reverting to the secondary manifest if reading one fails.
/// </summary>
/// <param name="backend">The backendwrapper to read from</param>
/// <param name="entry">The manifest to read</param>
/// <returns>The parsed manifest</returns>
private Manifestfile GetManifest(BackendWrapper backend, ManifestEntry entry)
{
if (m_options.DontReadManifests)
{
Manifestfile mf = new Manifestfile();
mf.SignatureHashes = null;
mf.ContentHashes = null;
return mf;
}
if (entry.ParsedManifest != null)
return entry.ParsedManifest;
else if (entry.Alternate != null && entry.Alternate.ParsedManifest != null)
return entry.Alternate.ParsedManifest;
if (OperationProgress != null && backend.Statistics != null)
OperationProgress(this, GetOperationType(), backend.Statistics.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusReadingManifest, entry.Time.ToShortDateString() + " " + entry.Time.ToShortTimeString()), "");
bool tryAlternateManifest = false;
//This method has some very special logic to ensure correct handling of errors
//The assumption is that it is possible to determine if the error occurred due to a
// transfer problem or a corrupt file. If the former happens, the operation should
// be retried, and thus an exception is thrown. If the latter, the file should
// be ignored and the backup file should be used.
//
//We detect a parsing error, either directly or indirectly through CryptographicException,
// and assume that a parsing error is an indication of a broken file.
//All other errors are assumed to be transfer problems, and throws exceptions.
//
//This holds as long as the backend always throws an exception if a partial file
// was downloaded. The FTP backend may not honor this, and some webservers
// may ommit the "Content-Length" header, which will cause problems.
//There is a guard agains partial downloads in BackendWrapper.GetInternal()
using (new Logging.Timer("Get " + entry.Filename))
using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
{
try
{
backend.Get(entry, null, tf, null);
//We now have the file decrypted, if the next step fails,
// its a broken xml or invalid content
tryAlternateManifest = true;
Manifestfile mf = new Manifestfile(tf, m_options.SkipFileHashChecks);
if (string.IsNullOrEmpty(mf.SelfFilename))
mf.SelfFilename = entry.Filename;
if (mf.ContentHashes != null && entry.Alternate != null)
{
//Special case, the manifest has not recorded all volumes,
// we must see if the alternate manifest has more volumes
if (entry.Volumes.Count > mf.ContentHashes.Count)
{
//Do not try the alternate, we just did
tryAlternateManifest = false;
Logging.Log.WriteMessage(string.Format(Strings.Interface.ReadingSecondaryManifestLogMessage, entry.Alternate.Filename), Duplicati.Library.Logging.LogMessageType.Information);
Manifestfile amf = null;
//Read the alternate file and try to differentiate between a defect file or a partial one
bool defectFile = false;
try
{
System.IO.File.Delete(tf);
backend.Get(entry.Alternate, null, tf, null);
}
catch (System.Security.Cryptography.CryptographicException cex)
{
//We assume that CryptoException means partial file
Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, cex), Duplicati.Library.Logging.LogMessageType.Warning);
defectFile = true;
}
if (!defectFile)
{
try
{
amf = new Manifestfile(tf, m_options.SkipFileHashChecks);
}
catch (Exception ex)
{
//Parsing error means partial file
Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, ex), Duplicati.Library.Logging.LogMessageType.Warning);
defectFile = true;
}
}
//If the alternate manifest is correct, assign it so we have a copy
if (!defectFile && amf != null)
{
if (string.IsNullOrEmpty(amf.SelfFilename))
amf.SelfFilename = entry.Alternate.Filename;
//If the alternate manifest has more files than the primary, we use that one
if (amf.ContentHashes != null && amf.ContentHashes.Count > mf.ContentHashes.Count)
{
entry.Alternate.ParsedManifest = amf;
if (m_options.SkipFileHashChecks)
{
mf.SignatureHashes = null;
mf.ContentHashes = null;
}
return amf;
}
}
}
}
if (m_options.SkipFileHashChecks)
{
mf.SignatureHashes = null;
mf.ContentHashes = null;
}
entry.ParsedManifest = mf;
return mf;
}
catch (Exception ex)
{
//Only try secondary if the parsing/decrypting fails, not if the transfer fails
if (entry.Alternate != null && (ex is System.Security.Cryptography.CryptographicException || tryAlternateManifest))
{
//TODO: If it is a version error, there is no need to read the alternate version
Logging.Log.WriteMessage(string.Format(Strings.Interface.PrimaryManifestReadErrorLogMessage, entry.Filename, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning);
try
{
Logging.Log.WriteMessage(string.Format(Strings.Interface.ReadingSecondaryManifestLogMessage, entry.Alternate.Filename), Duplicati.Library.Logging.LogMessageType.Information);
return GetManifest(backend, entry.Alternate);
}
catch (Exception ex2)
{
Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, ex2.Message), Duplicati.Library.Logging.LogMessageType.Warning);
}
}
//Report the original error
throw;
}
}
}
public List<KeyValuePair<BackupEntryBase, Exception>> VerifyBackupChain()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.Verify);
SetupCommonOptions(stats);
List<KeyValuePair<BackupEntryBase, Exception>> results = new List<KeyValuePair<BackupEntryBase, Exception>>();
if (m_options.DontReadManifests)
throw new InvalidOperationException(Strings.Interface.ManifestsMustBeRead);
if (m_options.SkipFileHashChecks)
throw new InvalidOperationException(Strings.Interface.CannotVerifyWithoutHashes);
if (!string.IsNullOrEmpty(m_options.SignatureCachePath))
{
stats.LogWarning(Strings.Interface.DisablingSignatureCacheForVerification, null);
m_options.SignatureCachePath = null;
}
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
{
//Find the spot in the chain where we start
ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime);
//Get the list of manifests to validate
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.Add(bestFit);
entries.AddRange(bestFit.Incrementals);
entries.Reverse();
foreach (ManifestEntry me in entries)
{
Manifestfile mf = null;
try
{
mf = GetManifest(backend, me);
VerifyBackupChainWithFiles(backend, me);
if (mf.SignatureHashes.Count != me.Volumes.Count)
results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, new Exception(string.Format(Strings.Interface.ManifestAndFileCountMismatchError, mf.SelfFilename, mf.SignatureHashes.Count, me.Volumes.Count))));
else
results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, null));
}
catch (Exception ex)
{
results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, ex));
}
if (mf != null)
{
int volumes = Math.Min(mf.SignatureHashes.Count, me.Volumes.Count);
for(int i = 0; i <volumes; i++)
{
if (m_options.Verificationlevel == VerificationLevel.Signature || m_options.Verificationlevel == VerificationLevel.Full)
{
try
{
using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
backend.Get(me.Volumes[i].Key, mf, tf, mf.SignatureHashes[i]);
results.Add(new KeyValuePair<BackupEntryBase, Exception>(me.Volumes[i].Key, null));
}
catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me.Volumes[i].Key, ex)); }
}
if (m_options.Verificationlevel == VerificationLevel.Full)
{
try
{
using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
backend.Get(me.Volumes[i].Value, mf, tf, mf.ContentHashes[i]);
results.Add(new KeyValuePair<BackupEntryBase, Exception>(me.Volumes[i].Value, null));
}
catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me.Volumes[i].Value, ex)); }
}
}
}
}
//Re-generate verification file
if (m_options.CreateVerificationFile)
{
//Stop any async operations
if (m_options.AsynchronousUpload)
backend.ExtractPendingUploads();
VerificationFile vf = new VerificationFile(entries, new FilenameStrategy(m_options));
using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
{
vf.Save(tf);
tf.Protected = true;
backend.Put(new VerificationEntry(entries[entries.Count - 1].Time), tf);
}
}
}
return results;
}
private void CreateFolder()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.CreateFolder);
SetupCommonOptions(stats);
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
backend.CreateFolder();
}
public string Backup(string[] sources)
{
BackupStatistics bs = new BackupStatistics(DuplicatiOperationMode.Backup);
SetupCommonOptions(bs);
BackendWrapper backend = null;
VerificationFile verification = null;
if (m_options.DontReadManifests)
throw new Exception(Strings.Interface.ManifestsMustBeReadOnBackups);
if (m_options.SkipFileHashChecks)
throw new Exception(Strings.Interface.CannotSkipHashChecksOnBackup);
if (sources == null || sources.Length == 0)
throw new Exception(Strings.Interface.NoSourceFoldersError);
//Make sure they all have the same format and exist
for (int i = 0; i < sources.Length; i++)
{
sources[i] = Utility.Utility.AppendDirSeparator(System.IO.Path.GetFullPath(sources[i]));
if (!System.IO.Directory.Exists(sources[i]))
throw new System.IO.IOException(String.Format(Strings.Interface.SourceFolderIsMissingError, sources[i]));
}
//Sanity check for duplicate folders and multiple inclusions of the same folder
for (int i = 0; i < sources.Length - 1; i++)
{
for (int j = i + 1; j < sources.Length; j++)
if (sources[i].Equals(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
throw new Exception(string.Format(Strings.Interface.SourceDirIsIncludedMultipleTimesError, sources[i]));
else if (sources[i].StartsWith(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
throw new Exception(string.Format(Strings.Interface.SourceDirsAreRelatedError, sources[i], sources[j]));
}
if (m_options.AsynchronousUpload)
{
m_asyncReserved = ASYNC_RESERVED;
m_allowUploadProgress = false;
}
//Unused, but triggers errors in the encryption setup here
Library.Interface.IEncryption encryptionModule = m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions);
using (new Logging.Timer("Backup from " + string.Join(";", sources) + " to " + m_backend))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, "");
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, "");
CheckLiveControl();
bool full = m_options.Full;
if (full)
bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseFlagWasSet, "full"));
backend = new BackendWrapper(bs, m_backend, m_options);
backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent);
backend.AsyncItemProcessedEvent += new EventHandler(backend_AsyncItemProcessedEvent);
m_progress = 0.0;
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, "");
CheckLiveControl();
List<ManifestEntry> backupsets;
if (full)
{
//This will create the target folder
backend.List(false);
backupsets = new List<ManifestEntry>();
}
else
{
//This will list all files on the backend and create the target folder
backupsets = backend.GetBackupSets();
}
if (backupsets.Count == 0)
{
if (!full)
bs.SetTypeReason(Strings.Interface.FullBecauseBackendIsEmpty);
full = true;
}
else
{
//A prioir backup exists, extract the compression and encryption modules used in the most recent entry
string compression = null;
string encryption = null;
for (int i = backupsets.Count - 1; compression == null && i >= 0; i--)
{
for (int j = backupsets[i].Incrementals.Count - 1; compression == null && j >= 0; j--)
for (int k = backupsets[i].Incrementals[j].Volumes.Count - 1; compression == null && k >= 0; k--)
{
compression = backupsets[i].Incrementals[j].Volumes[k].Key.Compression;
encryption = backupsets[i].Incrementals[j].Volumes[k].Key.EncryptionMode;
if (compression != null)
break;
}
for (int k = backupsets[i].Volumes.Count - 1; compression == null && k >= 0; k--)
{
compression = backupsets[i].Volumes[k].Key.Compression;
encryption = backupsets[i].Volumes[k].Key.EncryptionMode;
if (compression != null)
break;
}
}
if (compression != null)
{
m_options.SetEncryptionModuleDefault(encryption);
m_options.SetCompressionModuleDefault(compression);
}
}
string fullCriteria1 = null;
string fullCriteria2 = null;
if (!full)
{
full = DateTime.Now > m_options.FullIfOlderThan(backupsets[backupsets.Count - 1].Time);
if (full)
bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue));
else if (!string.IsNullOrEmpty(m_options.FullIfOlderThanValue))
fullCriteria1 = string.Format(Strings.Interface.IncrementalBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue);
}
if (!full && m_options.FullIfMoreThanNIncrementals > 0)
{
full = backupsets[backupsets.Count - 1].Incrementals.Count >= m_options.FullIfMoreThanNIncrementals;
if (full)
bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals));
else
fullCriteria2 = string.Format(Strings.Interface.IncrementalBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals);
}
bs.Full = full;
if (!full)
{
if (fullCriteria1 == null && fullCriteria2 == null)
bs.SetTypeReason(Strings.Interface.IncrementalBecauseNoFlagsWereSet);
else if (fullCriteria2 == null)
bs.SetTypeReason(fullCriteria1);
else if (fullCriteria1 == null)
bs.SetTypeReason(fullCriteria2);
else
bs.SetTypeReason(fullCriteria1 + ". " + fullCriteria2);
}
List<string> controlfiles = new List<string>();
if (!string.IsNullOrEmpty(m_options.SignatureControlFiles))
controlfiles.AddRange(m_options.SignatureControlFiles.Split(System.IO.Path.PathSeparator));
int vol = 0;
long totalsize = 0;
Manifestfile manifest = new Manifestfile();
using (Utility.TempFolder tempfolder = new Duplicati.Library.Utility.TempFolder())
{
List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = new List<KeyValuePair<ManifestEntry, Duplicati.Library.Interface.ICompression>>();
if (!full)
{
m_incrementalFraction = INCREMENAL_COST;
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.Add(backupsets[backupsets.Count - 1]);
entries.AddRange(backupsets[backupsets.Count - 1].Incrementals);
//Check before we start the download
CheckLiveControl();
VerifyBackupChainWithFiles(backend, entries[entries.Count - 1]);
if (m_options.CreateVerificationFile)
verification = new VerificationFile(entries, backend.FilenameStrategy);
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, "");
patches = FindPatches(backend, entries, tempfolder, false, bs);
//Check before we start the download
CheckLiveControl();
Manifestfile latest = GetManifest(backend, backupsets[backupsets.Count - 1]);
//Manifest version 1 does not support multiple folders
if (latest.Version == 1)
latest.SourceDirs = new string[] { sources[0] };
if (latest.SourceDirs.Length != sources.Length)
{
if (m_options.FullIfSourceFolderChanged)
{
Logging.Log.WriteMessage("Source folder count changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information);
if (!full)
bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged);
full = true;
}
else
throw new Exception(string.Format(Strings.Interface.NumberOfSourceFoldersHasChangedError, latest.SourceDirs.Length, sources.Length));
}
else
{
if (!m_options.AllowSourceFolderChange)
{
foreach (string s1 in latest.SourceDirs)
{
bool found = false;
foreach (string s2 in sources)
if (s1.Equals(s2, Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
{
found = true;
break;
}
if (!found)
{
if (m_options.FullIfSourceFolderChanged)
{
Logging.Log.WriteMessage("Source folders changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information);
if (!full)
bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged);
full = true;
break; //Exit the folder loop
}
else
throw new Exception(string.Format(Strings.Interface.SourceFoldersHasChangedError, s1));
}
}
manifest.SourceDirs = latest.SourceDirs;
}
else
{
manifest.SourceDirs = sources;
}
}
}
DateTime backuptime = DateTime.Now;
DateTime backupchaintime;
if (full)
{
patches.Clear();
m_incrementalFraction = 0.0;
manifest.SourceDirs = sources;
if (m_options.CreateVerificationFile)
verification = new VerificationFile(new ManifestEntry[0], backend.FilenameStrategy);
backupchaintime = backuptime;
}
else
{
backupchaintime = patches[0].Key.Time;
manifest.PreviousManifestFilename = patches[patches.Count - 1].Key.Filename;
manifest.PreviousManifestHash = patches[patches.Count - 1].Key.RemoteHash;
}
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusBuildingFilelist, "");
bool completedWithoutChanges;
using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(manifest.SourceDirs, bs, m_options.Filter, patches))
{
CheckLiveControl();
dir.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupRSyncDir_ProgressEvent);
dir.DisableFiletimeCheck = m_options.DisableFiletimeCheck;
dir.MaxFileSize = m_options.SkipFilesLargerThan;
using (new Logging.Timer("Initiating multipass"))
dir.InitiateMultiPassDiff(full, m_options);
string tempVolumeFolder = m_options.AsynchronousUpload ? m_options.AsynchronousUploadFolder : m_options.TempDir;
bool done = false;
while (!done && totalsize < m_options.MaxSize)
{
using (new Logging.Timer("Multipass " + (vol + 1).ToString()))
using (Utility.TempFile signaturefile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString())))
using (Utility.TempFile contentfile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString())))
{
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusCreatingVolume, vol + 1), "");
CheckLiveControl();
using (Library.Interface.ICompression signaturearchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, signaturefile, m_options.RawOptions))
using (Library.Interface.ICompression contentarchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, contentfile, m_options.RawOptions))
{
//If we are all out, stop now, this may cause incomplete partial files
if (m_options.MaxSize - totalsize < (contentarchive.FlushBufferSize + backend.FileSizeOverhead))
break;
//Add signature files to archive
foreach (string s in controlfiles)
if (!string.IsNullOrEmpty(s))
using (System.IO.Stream cs = signaturearchive.CreateFile(System.IO.Path.Combine(RSync.RSyncDir.CONTROL_ROOT, System.IO.Path.GetFileName(s))))
using (System.IO.FileStream fs = System.IO.File.OpenRead(s))
Utility.Utility.CopyStream(fs, cs);
//Only add control files to the very first volume
controlfiles.Clear();
done = dir.MakeMultiPassDiff(signaturearchive, contentarchive, (Math.Min(m_options.VolumeSize, m_options.MaxSize - totalsize)) - backend.FileSizeOverhead);
//TODO: This is not the correct size, we need to account for file size overhead as well
totalsize += signaturearchive.Size;
totalsize += contentarchive.Size;
//TODO: This is not the best way to determine this
if (totalsize >= m_options.MaxSize)
dir.FinalizeMultiPass(signaturearchive, contentarchive, long.MaxValue);
}
completedWithoutChanges = done && !dir.AnyChangesFound;
if (m_options.UploadUnchangedBackups || full)
completedWithoutChanges = false;
if (!completedWithoutChanges)
{
if (m_options.AsynchronousUpload)
{
m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload;
m_allowUploadProgress = true;
m_allowUploadProgressAfter = DateTime.Now.AddSeconds(1);
}
else
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingContentVolume, vol + 1), "");
//Last check before we upload, we do not interrupt transfers
CheckLiveControl();
//The backendwrapper will remove these
signaturefile.Protected = true;
contentfile.Protected = true;
ContentEntry ce = new ContentEntry(backuptime, full, vol + 1);
SignatureEntry se = new SignatureEntry(backuptime, full, vol + 1);
using (new Logging.Timer("Writing delta file " + (vol + 1).ToString()))
backend.Put(ce, contentfile);
if (!m_options.AsynchronousUpload)
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingSignatureVolume, vol + 1), "");
using (new Logging.Timer("Writing remote signatures"))
backend.Put(se, signaturefile);
manifest.AddEntries(ce, se);
if (verification != null)
{
verification.AddFile(ce);
verification.AddFile(se);
}
}
}
if (!completedWithoutChanges)
{
//The backend wrapper will remove these
Utility.TempFile mf = new Duplicati.Library.Utility.TempFile();
using (new Logging.Timer("Writing manifest " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK")))
{
//Alternate primary/secondary
ManifestEntry mfe = new ManifestEntry(backuptime, full, manifest.SignatureHashes.Count % 2 != 0);
manifest.SelfFilename = backend.GenerateFilename(mfe);
manifest.Save(mf);
if (!m_options.AsynchronousUpload)
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingManifestVolume, vol + 1), "");
//Write the file
mf.Protected = true;
backend.Put(mfe, mf);
if (verification != null)
verification.UpdateManifest(mfe);
}
if (verification != null)
{
using (new Logging.Timer("Writing verification " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK")))
{
Utility.TempFile vt = new Duplicati.Library.Utility.TempFile();
verification.Save(vt);
if (!m_options.AsynchronousUpload)
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusUploadingVerificationVolume, "");
vt.Protected = true;
backend.Put(new VerificationEntry(backupchaintime), vt);
}
}
if (m_options.AsynchronousUpload)
m_allowUploadProgress = false;
//The file volume counter
vol++;
}
}
}
//If we are running asynchronous, we now enter the end-game
if (m_options.AsynchronousUpload)
{
m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload;
m_allowUploadProgress = true;
m_allowUploadProgressAfter = DateTime.Now;
//Before we clear the temp folder, we need to ensure that all volumes are uploaded.
//To allow the UI to show some progress while uploading, we perform the remaining
// uploads synchronous
List<KeyValuePair<BackupEntryBase, string>> pendingUploads = backend.ExtractPendingUploads();
//Figure out what volume number we are at
foreach (KeyValuePair<BackupEntryBase, string> p in pendingUploads)
if (p.Key is ManifestEntry)
vol--;
double unitcost = m_asyncReserved / pendingUploads.Count;
//The upload each remaining volume in order
foreach (KeyValuePair<BackupEntryBase, string> p in pendingUploads)
{
string msg;
if (p.Key is ManifestEntry)
{
vol++;
msg = string.Format(Strings.Interface.StatusUploadingManifestVolume, vol);
}
else if (p.Key is SignatureEntry)
msg = string.Format(Strings.Interface.StatusUploadingSignatureVolume, ((SignatureEntry)p.Key).Volumenumber);
else if (p.Key is ContentEntry)
{
msg = string.Format(Strings.Interface.StatusUploadingContentVolume, ((ContentEntry)p.Key).Volumenumber);
//We allow a stop or pause request here
CheckLiveControl();
}
else if (p.Key is VerificationEntry)
msg = Strings.Interface.StatusUploadingVerificationVolume;
else
throw new InvalidOperationException();
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, msg, "");
backend.Put(p.Key, p.Value);
m_asyncReserved -= unitcost;
m_progress += unitcost;
}
}
}
}
catch(Exception ex)
{
//If this is a controlled user-requested stop, wait for the current upload to complete
if (backend != null && ex is LiveControl.ExecutionStoppedException)
{
try
{
if (m_options.AsynchronousUpload)
{
m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload;
m_allowUploadProgress = true;
m_allowUploadProgressAfter = DateTime.Now;
//Wait for the current upload to complete and then delete all remaining temporary files
foreach (KeyValuePair<BackupEntryBase, string> p in backend.ExtractPendingUploads())
try
{
if (System.IO.File.Exists(p.Value))
System.IO.File.Delete(p.Value);
}
catch { } //Better to delete as many as possible rather than choke on a single file
}
}
catch { } //We already have an exception, just go with that
}
if (backend == null || backend.ManifestUploads == 0)
{
Logging.Log.WriteMessage(string.Format(Strings.Interface.ErrorRunningBackup, ex.Message), Logging.LogMessageType.Error);
throw; //This also activates "finally", unlike in other languages...
}
Logging.Log.WriteMessage(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), Logging.LogMessageType.Warning);
bs.LogError(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), ex);
}
finally
{
m_progress = 100.0;
if (backend != null)
try { backend.Dispose(); }
catch { }
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
}
bs.EndTime = DateTime.Now;
return bs.ToString();
}
public string DeleteAllButNFull()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButNFull);
SetupCommonOptions(stats);
int x = Math.Max(0, m_options.DeleteAllButNFull);
StringBuilder sb = new StringBuilder();
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
List<ManifestEntry> entries = backend.GetBackupSets();
List<ManifestEntry> toremove = new List<ManifestEntry>();
while (entries.Count > x)
{
if (entries.Count == 1 && !m_options.AllowFullRemoval)
{
sb.AppendLine(string.Format(Strings.Interface.NotDeletingLastFullMessage, entries[0].Time));
break;
}
ManifestEntry be = entries[0];
entries.RemoveAt(0);
be.Incrementals.Reverse();
toremove.AddRange(be.Incrementals);
toremove.Add(be);
}
if (entries.Count == 0 && toremove.Count > 0 && !m_options.AllowFullRemoval)
throw new Exception(Strings.Interface.InternalDeleteCountError);
sb.Append(RemoveBackupSets(backend, toremove));
}
finally
{
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
return sb.ToString();
}
public IList<string> ListCurrentFiles()
{
RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.ListCurrentFiles);
SetupCommonOptions(rs);
Utility.FilenameFilter filter = m_options.Filter;
DateTime timelimit = m_options.RestoreTime;
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.List, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
List<string> res;
using (BackendWrapper backend = new BackendWrapper(rs, m_backend, m_options))
using (Utility.TempFolder basefolder = new Duplicati.Library.Utility.TempFolder())
{
ManifestEntry bestFit = backend.GetBackupSet(timelimit);
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.Add(bestFit);
entries.AddRange(bestFit.Incrementals);
List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = FindPatches(backend, entries, basefolder, false, rs);
using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(new string[] { basefolder }, rs, filter, patches))
res = dir.UnmatchedFiles();
}
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.List, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
return res;
}
public string[] ListSourceFolders()
{
RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.ListSourceFolders);
SetupCommonOptions(rs);
if (m_options.DontReadManifests)
throw new Exception(Strings.Interface.ManifestsMustBeRead);
DateTime timelimit = m_options.RestoreTime;
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.List, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
string[] res;
using (BackendWrapper backend = new BackendWrapper(rs, m_backend, m_options))
using (Utility.TempFile mfile = new Duplicati.Library.Utility.TempFile())
{
ManifestEntry bestFit = backend.GetBackupSet(timelimit);
backend.Get(bestFit, null, mfile, null);
res = new Manifestfile(mfile, m_options.SkipFileHashChecks).SourceDirs;
}
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.List, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
return res;
}
public string[] List()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.List);
SetupCommonOptions(stats);
List<string> res = new List<string>();
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.List, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
foreach (Duplicati.Library.Interface.IFileEntry fe in backend.List(false))
res.Add(fe.Name);
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.List, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
return res.ToArray();
}
}
public List<KeyValuePair<RSync.RSyncDir.PatchFileType, string>> ListActualSignatureFiles()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.ListActualSignatureFiles);
SetupCommonOptions(stats);
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
{
ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime);
if (bestFit.Incrementals.Count > 0) //Get the most recent incremental
bestFit = bestFit.Incrementals[bestFit.Incrementals.Count - 1];
using (Utility.TempFolder folder = new Duplicati.Library.Utility.TempFolder())
{
List<Library.Interface.ICompression> patches = new List<Duplicati.Library.Interface.ICompression>();
foreach (KeyValuePair<ManifestEntry, Library.Interface.ICompression> entry in FindPatches(backend, new List<ManifestEntry>(new ManifestEntry[] { bestFit }), folder, false, stats))
patches.Add(entry.Value);
using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(new string[] { folder }, stats, null))
return dir.ListPatchFiles(patches);
}
}
}
public List<ManifestEntry> GetBackupSets()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.GetBackupSets);
SetupCommonOptions(stats);
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
return backend.GetBackupSets();
}
/// <summary>
/// Reads through a backup and finds the last backup entry that has a specific file
/// </summary>
/// <returns></returns>
public List<KeyValuePair<string, DateTime>> FindLastFileVersion()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.FindLastFileVersion);
SetupCommonOptions(stats);
if (m_options.DontReadManifests)
throw new Exception(Strings.Interface.ManifestsMustBeRead);
if (string.IsNullOrEmpty(m_options.FileToRestore))
throw new Exception(Strings.Interface.NoFilesGivenError);
string[] filesToFind = m_options.FileToRestore.Split(System.IO.Path.PathSeparator);
KeyValuePair<string, DateTime>[] results = new KeyValuePair<string, DateTime>[filesToFind.Length];
for (int i = 0; i < results.Length; i++)
results[i] = new KeyValuePair<string, DateTime>(filesToFind[i], new DateTime(0));
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
{
//Extract the full backup set list
List<ManifestEntry> fulls = backend.GetBackupSets();
//Flatten the list
List<ManifestEntry> workList = new List<ManifestEntry>();
//The list is oldest first, this function work newest first
fulls.Reverse();
foreach (ManifestEntry f in fulls)
{
f.Incrementals.Reverse();
workList.AddRange(f.Incrementals);
workList.Add(f);
}
bool warned_manifest_v1 = false;
foreach (ManifestEntry mf in workList)
{
List<Manifestfile.HashEntry> signatureHashes = null;
Manifestfile mfi;
using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
{
backend.Get(mf, null, tf, null);
mfi = new Manifestfile(tf, m_options.SkipFileHashChecks);
if (!m_options.SkipFileHashChecks)
signatureHashes = mfi.SignatureHashes;
}
//If there are no volumes, don't stop here
bool any_unmatched = true;
if (stats != null && !warned_manifest_v1 && (mfi.SourceDirs == null || mfi.SourceDirs.Length == 0))
{
warned_manifest_v1 = true;
stats.LogWarning(Strings.Interface.ManifestVersionRequiresRelativeNamesWarning, null);
}
foreach(KeyValuePair<SignatureEntry, ContentEntry> e in mf.Volumes)
using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
{
//Skip non-approved signature files
if (signatureHashes != null && e.Key.Volumenumber > signatureHashes.Count)
{
stats.LogWarning(string.Format(Strings.Interface.SkippedUnlistedSignatureFileWarning, e.Key.Filename), null);
continue;
}
backend.Get(e.Key, mfi, tf, signatureHashes == null ? null : signatureHashes[e.Key.Volumenumber - 1]);
any_unmatched = false;
RSync.RSyncDir.ContainsFile(mfi, filesToFind, DynamicLoader.CompressionLoader.GetModule(e.Key.Compression, tf, m_options.RawOptions));
for (int i = 0; i < filesToFind.Length; i++)
{
if (results[i].Value.Ticks == 0 && string.IsNullOrEmpty(filesToFind[i]))
results[i] = new KeyValuePair<string,DateTime>(results[i].Key, mf.Time);
else
any_unmatched = true;
}
if (!any_unmatched)
break;
}
if (!any_unmatched)
break;
}
return new List<KeyValuePair<string,DateTime>>(results);
}
}
public string DeleteOlderThan()
{
StringBuilder sb = new StringBuilder();
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteOlderThan);
SetupCommonOptions(stats);
DateTime expires = m_options.RemoveOlderThan;
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
List<ManifestEntry> entries = backend.GetBackupSets();
List<ManifestEntry> toremove = new List<ManifestEntry>();
while (entries.Count > 0 && entries[0].Time <= expires)
{
if (entries.Count == 1 && !m_options.AllowFullRemoval)
{
sb.AppendLine(string.Format(Strings.Interface.NotDeletingLastFullMessage, entries[0].Time));
break;
}
ManifestEntry be = entries[0];
entries.RemoveAt(0);
bool hasNewer = false;
foreach (ManifestEntry bex in be.Incrementals)
if (bex.Time >= expires)
{
hasNewer = true;
break;
}
if (hasNewer)
{
List<ManifestEntry> t = new List<ManifestEntry>(be.Incrementals);
t.Insert(0, be);
for (int i = 0; i < t.Count; i++)
if (t[i].Time <= expires)
sb.AppendLine(string.Format(Strings.Interface.NotDeletingBackupSetMessage, t[i].Time.ToString(System.Globalization.CultureInfo.InvariantCulture)));
break;
}
else
{
be.Incrementals.Reverse();
toremove.AddRange(be.Incrementals);
toremove.Add(be);
}
}
if (entries.Count == 0 && toremove.Count > 0 && !m_options.AllowFullRemoval)
throw new Exception(Strings.Interface.InternalDeleteCountError);
sb.Append(RemoveBackupSets(backend, toremove));
}
finally
{
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
return sb.ToString();
}
/// <summary>
/// Verifies the backup chain for producing a new backup on top.
/// This will check that all files are accounted for in the file list.
/// </summary>
/// <param name="entry">The newest entry to check</param>
private void VerifyBackupChainWithFiles(BackendWrapper backend, ManifestEntry entry)
{
VerifyManifestChain(backend, entry);
string errorMessage = Environment.NewLine + Strings.Interface.DeleteManifestsSuggestion + Environment.NewLine + Environment.NewLine;
while (entry != null)
{
Manifestfile parsed = GetManifest(backend, entry);
errorMessage += entry.Filename + Environment.NewLine;
if (entry.Volumes.Count != parsed.SignatureHashes.Count || entry.Volumes.Count != parsed.ContentHashes.Count)
{
//If we have an extra set, the connection could have died right before the manifest was uploaded
if (parsed.SignatureHashes.Count == parsed.ContentHashes.Count && entry.Volumes.Count - 1 == parsed.ContentHashes.Count)
{
backend.AddOrphan(entry.Volumes[entry.Volumes.Count - 1].Value);
backend.AddOrphan(entry.Volumes[entry.Volumes.Count - 1].Key);
entry.Volumes.RemoveAt(entry.Volumes.Count - 1);
}
else
{
throw new Exception(
string.Format(Strings.Interface.ManifestAndFileCountMismatchError, entry.Filename, parsed.SignatureHashes.Count, entry.Volumes.Count)
+ errorMessage
);
}
}
for(int i = 0; i < entry.Volumes.Count; i++)
{
if (entry.Volumes[i].Key.Filesize > 0 && parsed.SignatureHashes[i].Size > 0 && entry.Volumes[i].Key.Filesize != parsed.SignatureHashes[i].Size)
throw new Exception(
string.Format(Strings.Interface.FileSizeMismatchError, entry.Volumes[i].Key.Filename, entry.Volumes[i].Key.Filesize, parsed.SignatureHashes[i].Size)
+ errorMessage
);
if (entry.Volumes[i].Value.Filesize >= 0 && parsed.ContentHashes[i].Size >= 0 && entry.Volumes[i].Value.Filesize != parsed.ContentHashes[i].Size)
throw new Exception(
string.Format(Strings.Interface.FileSizeMismatchError, entry.Volumes[i].Value.Filename, entry.Volumes[i].Value.Filesize, parsed.ContentHashes[i].Size)
+ errorMessage
);
if (!string.IsNullOrEmpty(parsed.SignatureHashes[i].Name) && !parsed.SignatureHashes[i].Name.Equals(entry.Volumes[i].Key.Fileentry.Name, StringComparison.InvariantCultureIgnoreCase))
throw new Exception(
string.Format(Strings.Interface.FilenameMismatchError, parsed.SignatureHashes[i].Name, entry.Volumes[i].Key.Fileentry.Name)
+ errorMessage
);
if (!string.IsNullOrEmpty(parsed.ContentHashes[i].Name) && !parsed.ContentHashes[i].Name.Equals(entry.Volumes[i].Value.Fileentry.Name, StringComparison.InvariantCultureIgnoreCase))
throw new Exception(
string.Format(Strings.Interface.FilenameMismatchError, parsed.ContentHashes[i].Name, entry.Volumes[i].Value.Fileentry.Name)
+ errorMessage
);
}
entry = entry.Previous;
}
}
public string Restore(string[] target)
{
RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.Restore);
SetupCommonOptions(rs);
m_progress = 0;
BackendWrapper backend = null;
m_restorePatches = 0;
using (new Logging.Timer("Restore from " + m_backend + " to " + string.Join(System.IO.Path.PathSeparator.ToString(), target)))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Restore, rs.OperationMode, -1, -1, Strings.Interface.StatusStarted, "");
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, -1, -1, Strings.Interface.StatusStarted, "");
Utility.FilenameFilter filter = m_options.Filter;
//Filter is prefered, if both file and filter is specified
if (!m_options.HasFilter && !string.IsNullOrEmpty(m_options.FileToRestore))
{
List<Utility.IFilenameFilter> list = new List<Duplicati.Library.Utility.IFilenameFilter>();
list.Add(new Utility.FilelistFilter(true, m_options.FileToRestore.Split(System.IO.Path.PathSeparator)));
list.Add(new Utility.RegularExpressionFilter(false, ".*"));
filter = new Duplicati.Library.Utility.FilenameFilter(list);
}
backend = new BackendWrapper(rs, m_backend, m_options);
backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent);
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, "");
ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime);
//We will need all the manifests downloaded anyway
if (!m_options.DontReadManifests)
{
if (bestFit.Incrementals.Count > 0)
VerifyManifestChain(backend, bestFit.Incrementals[bestFit.Incrementals.Count - 1]);
else
VerifyManifestChain(backend, bestFit);
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, "");
}
m_progress = INCREMENAL_COST;
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.Add(bestFit);
entries.AddRange(bestFit.Incrementals);
int patchno = 0;
foreach (ManifestEntry be in entries)
m_restorePatches += be.Volumes.Count;
Manifestfile rootManifest = GetManifest(backend, bestFit);
int sourceDirCount = (rootManifest.SourceDirs == null || rootManifest.SourceDirs.Length == 0) ? 1 : rootManifest.SourceDirs.Length;
//After reading the first manifest, we know the source folder count
if ((rootManifest.SourceDirs == null || rootManifest.SourceDirs.Length == 0) && target.Length > 1)
{
//V1 support
rs.LogWarning(string.Format(Strings.Interface.TooManyTargetFoldersWarning, 1, target.Length), null);
Array.Resize(ref target, 1);
}
else if (target.Length > sourceDirCount)
{
//If we get too many, we can just cut them off
rs.LogWarning(string.Format(Strings.Interface.TooManyTargetFoldersWarning, sourceDirCount, target.Length), null);
Array.Resize(ref target, rootManifest.SourceDirs.Length);
}
else if (target.Length != 1 && target.Length < sourceDirCount)
{
//If we get too few, we have to bail
throw new Exception(string.Format(Strings.Interface.TooFewTargetFoldersError, sourceDirCount, target.Length));
}
else if (target.Length == 1 && sourceDirCount > 1)
{
//If there is just one target folder, we automatically compose target subfolders
string[] newtargets = new string[rootManifest.SourceDirs.Length];
List<string> suggestions = new List<string>();
for (int i = 0; i < rootManifest.SourceDirs.Length; i++)
{
string s = rootManifest.SourceDirs[i];
//HACK: We use a leading / in the path name to detect source OS
// all paths are absolute, so this detects all unix like systems
string dirSepChar = s.StartsWith("/") ? "/" : "\\";
if (s.EndsWith(dirSepChar))
s = s.Substring(0, s.Length - 1);
int lix = s.LastIndexOf(dirSepChar);
if (lix < 0 || lix + 1 >= s.Length)
s = i.ToString();
else
s = s.Substring(lix + 1);
foreach (char c in System.IO.Path.GetInvalidFileNameChars())
s = s.Replace(c, '_');
suggestions.Add(s);
}
Dictionary<string, int> duplicates = new Dictionary<string, int>(Library.Utility.Utility.ClientFilenameStringComparer);
for (int i = 0; i < suggestions.Count; i++)
if (duplicates.ContainsKey(suggestions[i]))
duplicates[suggestions[i]]++;
else
duplicates[suggestions[i]] = 1;
for (int i = 0; i < newtargets.Length; i++)
{
string suffix = duplicates[suggestions[i]] > 1 ? i.ToString() : suggestions[i];
newtargets[i] = System.IO.Path.Combine(target[0], suffix);
}
target = newtargets;
}
//Make sure all targets exist
foreach(string s in target)
if (!System.IO.Directory.Exists(s))
System.IO.Directory.CreateDirectory(s);
using (RSync.RSyncDir sync = new Duplicati.Library.Main.RSync.RSyncDir(target, rs, filter))
{
sync.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(RestoreRSyncDir_ProgressEvent);
foreach (ManifestEntry be in entries)
{
m_progress = ((1.0 - INCREMENAL_COST) * (patchno / (double)m_restorePatches)) + INCREMENAL_COST;
CheckLiveControl();
Manifestfile manifest = be == bestFit ? rootManifest : GetManifest(backend, be);
CheckLiveControl();
foreach (KeyValuePair<SignatureEntry, ContentEntry> vol in be.Volumes)
{
ContentEntry contentVol = vol.Value;
SignatureEntry signatureVol = vol.Key;
m_progress = ((1.0 - INCREMENAL_COST) * (patchno / (double)m_restorePatches)) + INCREMENAL_COST;
//Skip nonlisted
if (manifest.ContentHashes != null && contentVol.Volumenumber > manifest.ContentHashes.Count)
{
Logging.Log.WriteMessage(string.Format(Strings.Interface.SkippedContentVolumeLogMessage, contentVol.Volumenumber), Duplicati.Library.Logging.LogMessageType.Warning);
rs.LogWarning(string.Format(Strings.Interface.SkippedContentVolumeLogMessage, contentVol.Volumenumber), null);
patchno++;
continue;
}
using (Utility.TempFile patchzip = new Duplicati.Library.Utility.TempFile())
{
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusPatching, patchno + 1), "");
CheckLiveControl();
if (m_options.HasFilter || !string.IsNullOrEmpty(m_options.FileToRestore))
{
bool hasFiles = false;
using (Utility.TempFile sigFile = new Duplicati.Library.Utility.TempFile())
{
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusDownloadingSignatureVolume, patchno + 1), "");
try
{
using (new Logging.Timer("Get " + signatureVol.Filename))
backend.Get(signatureVol, manifest, sigFile, manifest.SignatureHashes == null ? null : manifest.SignatureHashes[signatureVol.Volumenumber - 1]);
}
catch (BackendWrapper.HashMismathcException hme)
{
hasFiles = true;
rs.LogError(string.Format(Strings.Interface.FileHashFailure, hme.Message), hme);
}
if (!hasFiles)
using (Library.Interface.ICompression patch = DynamicLoader.CompressionLoader.GetModule(signatureVol.Compression, sigFile, m_options.RawOptions))
{
foreach(KeyValuePair<RSync.RSyncDir.PatchFileType, string> k in sync.ListPatchFiles(patch))
if (filter.ShouldInclude("", System.IO.Path.DirectorySeparatorChar.ToString() + k.Value))
{
//TODO: Perhaps a bit much to download the content archive
// if the file is only marked for deletion?
hasFiles = true;
break;
}
}
}
if (!hasFiles)
{
//Avoid downloading the content file
patchno++;
continue;
}
}
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusDownloadingContentVolume, patchno + 1), "");
using (new Logging.Timer("Get " + contentVol.Filename))
backend.Get(contentVol, manifest, patchzip, manifest.ContentHashes == null ? null : manifest.ContentHashes[contentVol.Volumenumber - 1]);
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusPatching, patchno + 1), "");
using (new Logging.Timer((patchno == 0 ? "Full restore to: " : "Incremental restore " + patchno.ToString() + " to: ") + string.Join(System.IO.Path.PathSeparator.ToString(), target)))
using (Library.Interface.ICompression patch = DynamicLoader.CompressionLoader.GetModule(contentVol.Compression, patchzip, m_options.RawOptions))
sync.Patch(target, patch);
}
patchno++;
}
//Make sure there are no partial files, as partial files are not allowed to span backup sets
sync.FinalizeRestore();
}
}
}
finally
{
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
}
rs.EndTime = DateTime.Now;
return rs.ToString();
}
/// <summary>
/// Validates the manifest chain, starting by evaluating the current manifest and going backwards in the chain
/// </summary>
/// <param name="backend">The backend wrapper</param>
/// <param name="entry">The entry to verify</param>
private void VerifyManifestChain(BackendWrapper backend, ManifestEntry entry)
{
if (m_options.DontReadManifests)
throw new InvalidOperationException(Strings.Interface.CannotVerifyChain);
while (entry.Previous != null)
{
Manifestfile parsed = GetManifest(backend, entry);
//If this manifest is not version 3, the chain verification stops here
if (parsed.Version < 3)
return;
ManifestEntry previous = entry.Previous;
if (entry.Previous.Alternate != null && entry.Previous.Alternate.Filename == parsed.PreviousManifestFilename)
previous = entry.Previous.Alternate;
if (parsed.PreviousManifestFilename != previous.Filename)
throw new System.IO.InvalidDataException(string.Format(Strings.Interface.PreviousManifestFilenameMismatchError, entry.Filename, parsed.PreviousManifestFilename, previous.Filename));
if (!m_options.SkipFileHashChecks)
{
//Load the Remotehash property
GetManifest(backend, previous);
if (parsed.PreviousManifestHash != previous.RemoteHash)
throw new System.IO.InvalidDataException(string.Format(Strings.Interface.PreviousManifestHashMismatchError, entry.Filename, parsed.PreviousManifestHash, previous.RemoteHash));
}
entry = previous;
}
}
/// <summary>
/// Restores control files added to a backup.
/// </summary>
/// <param name="target">The folder into which to restore the files</param>
/// <returns>A restore report</returns>
public string RestoreControlFiles(string target)
{
RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.RestoreControlfiles);
SetupCommonOptions(rs);
BackendWrapper backend = null;
using (new Logging.Timer("Restore control files from " + m_backend + " to " + target))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
backend = new BackendWrapper(rs, m_backend, m_options);
List<ManifestEntry> attempts = backend.GetBackupSets();
List<ManifestEntry> flatlist = new List<ManifestEntry>();
foreach (ManifestEntry be in attempts)
{
flatlist.Add(be);
flatlist.AddRange(be.Incrementals);
}
flatlist.Reverse();
string prefix = Utility.Utility.AppendDirSeparator(RSync.RSyncDir.CONTROL_ROOT);
foreach (ManifestEntry be in flatlist)
{
if (be.Volumes.Count > 0)
using(Utility.TempFile z = new Duplicati.Library.Utility.TempFile())
{
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, string.Format(Strings.Interface.StatusReadingIncrementalFile, be.Volumes[0].Key.Filename), "");
Manifestfile mf = GetManifest(backend, be);
OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, string.Format(Strings.Interface.StatusReadingIncrementalFile, be.Volumes[0].Key.Filename), "");
using (new Logging.Timer("Get " + be.Volumes[0].Key.Filename))
backend.Get(be.Volumes[0].Key, mf, z, mf.SignatureHashes == null ? null : mf.SignatureHashes[0]);
using(Library.Interface.ICompression fz = DynamicLoader.CompressionLoader.GetModule(be.Volumes[0].Key.Compression, z, m_options.RawOptions))
{
bool any = false;
foreach (string f in fz.ListFiles(prefix))
{
any = true;
using (System.IO.Stream s1 = fz.OpenRead(f))
using (System.IO.Stream s2 = System.IO.File.Create(System.IO.Path.Combine(target, f.Substring(prefix.Length))))
Utility.Utility.CopyStream(s1, s2);
}
if (any)
break;
rs.LogError(string.Format(Strings.Interface.FailedToFindControlFilesMessage, be.Volumes[0].Key.Filename), null);
}
}
}
}
finally
{
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
}
rs.EndTime = DateTime.Now;
return rs.ToString();
}
public string Cleanup()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.CleanUp);
SetupCommonOptions(stats);
bool anyRemoved = false;
StringBuilder sb = new StringBuilder();
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
{
List<ManifestEntry> sorted = backend.GetBackupSets();
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.AddRange(sorted);
foreach (ManifestEntry be in sorted)
entries.AddRange(be.Incrementals);
string cleanup = backend.DeleteOrphans(false);
if (!string.IsNullOrEmpty(cleanup))
sb.AppendLine(cleanup);
if (m_options.SkipFileHashChecks)
throw new Exception(Strings.Interface.CannotCleanWithoutHashesError);
if (m_options.DontReadManifests)
throw new Exception(Strings.Interface.CannotCleanWithoutHashesError);
//We need the manifests anyway, so we verify the chain
if (entries.Count > 0)
VerifyManifestChain(backend, entries[0]);
//Now compare the actual filelist with the manifest
foreach (ManifestEntry be in entries)
{
Manifestfile manifest = GetManifest(backend, be);
int count = manifest.ContentHashes.Count;
for (int i = count - 1; i < be.Volumes.Count; i++)
{
anyRemoved = true;
string sigmsg = string.Format(Strings.Interface.RemovingPartialFilesMessage, be.Volumes[i].Key.Filename);
string cntmsg = string.Format(Strings.Interface.RemovingPartialFilesMessage, be.Volumes[i].Value.Filename);
Logging.Log.WriteMessage(sigmsg, Duplicati.Library.Logging.LogMessageType.Information);
Logging.Log.WriteMessage(cntmsg, Duplicati.Library.Logging.LogMessageType.Information);
sb.AppendLine(sigmsg);
sb.AppendLine(cntmsg);
if (m_options.Force)
{
backend.Delete(be.Volumes[i].Key);
backend.Delete(be.Volumes[i].Value);
}
}
}
}
if (!m_options.Force && anyRemoved)
{
Logging.Log.WriteMessage(Strings.Interface.FilesAreNotForceDeletedMessage, Duplicati.Library.Logging.LogMessageType.Information);
sb.AppendLine(Strings.Interface.FilesAreNotForceDeletedMessage);
}
return sb.ToString(); //TODO: Write a message here?
}
public string DeleteAllButN()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButN);
SetupCommonOptions(stats);
int x = Math.Max(0, m_options.DeleteAllButNFull);
StringBuilder sb = new StringBuilder();
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, "");
List<ManifestEntry> flatlist = new List<ManifestEntry>();
List<ManifestEntry> entries = backend.GetBackupSets();
//Get all backups as a flat list
foreach (ManifestEntry me in entries)
{
flatlist.Add(me);
flatlist.AddRange(me.Incrementals);
}
//Now remove all but those requested
List<ManifestEntry> toremove = new List<ManifestEntry>();
while (flatlist.Count > x)
{
toremove.Add(flatlist[0]);
flatlist.RemoveAt(0);
}
//If there are still chains left, make sure we do not end up with a partial chain
if (!m_options.AllowFullRemoval || flatlist.Count != 0)
{
//Go back until we have a full chain
while (toremove.Count > 0 && (flatlist.Count == 0 || !flatlist[0].IsFull))
{
sb.AppendLine(string.Format(Strings.Interface.NotDeletingBackupSetMessage, toremove[toremove.Count - 1].Time));
flatlist.Insert(0, toremove[toremove.Count - 1]);
toremove.RemoveAt(toremove.Count - 1);
}
}
if (toremove.Count > 0 && !m_options.AllowFullRemoval && (flatlist.Count == 0 || !flatlist[0].IsFull))
throw new Exception(Strings.Interface.InternalDeleteCountError);
sb.Append(RemoveBackupSets(backend, toremove));
}
finally
{
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
return sb.ToString();
}
