public string Backup(string[] sources)
{
BackupStatistics bs = new BackupStatistics(DuplicatiOperationMode.Backup);
SetupCommonOptions(bs);
BackendWrapper backend = null;
VerificationFile verification = null;
if (m_options.DontReadManifests)
throw new Exception(Strings.Interface.ManifestsMustBeReadOnBackups);
if (m_options.SkipFileHashChecks)
throw new Exception(Strings.Interface.CannotSkipHashChecksOnBackup);
if (sources == null || sources.Length == 0)
throw new Exception(Strings.Interface.NoSourceFoldersError);
//Make sure they all have the same format and exist
for (int i = 0; i < sources.Length; i++)
{
sources[i] = Utility.Utility.AppendDirSeparator(System.IO.Path.GetFullPath(sources[i]));
if (!System.IO.Directory.Exists(sources[i]))
throw new System.IO.IOException(String.Format(Strings.Interface.SourceFolderIsMissingError, sources[i]));
}
//Sanity check for duplicate folders and multiple inclusions of the same folder
for (int i = 0; i < sources.Length - 1; i++)
{
for (int j = i + 1; j < sources.Length; j++)
if (sources[i].Equals(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
throw new Exception(string.Format(Strings.Interface.SourceDirIsIncludedMultipleTimesError, sources[i]));
else if (sources[i].StartsWith(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
throw new Exception(string.Format(Strings.Interface.SourceDirsAreRelatedError, sources[i], sources[j]));
}
if (m_options.AsynchronousUpload)
{
m_asyncReserved = ASYNC_RESERVED;
m_allowUploadProgress = false;
}
//Unused, but triggers errors in the encryption setup here
Library.Interface.IEncryption encryptionModule = m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions);
using (new Logging.Timer("Backup from " + string.Join(";", sources) + " to " + m_backend))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, "");
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, "");
CheckLiveControl();
bool full = m_options.Full;
if (full)
bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseFlagWasSet, "full"));
backend = new BackendWrapper(bs, m_backend, m_options);
backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent);
backend.AsyncItemProcessedEvent += new EventHandler(backend_AsyncItemProcessedEvent);
m_progress = 0.0;
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, "");
CheckLiveControl();
List<ManifestEntry> backupsets;
if (full)
{
//This will create the target folder
backend.List(false);
backupsets = new List<ManifestEntry>();
}
else
{
//This will list all files on the backend and create the target folder
backupsets = backend.GetBackupSets();
}
if (backupsets.Count == 0)
{
if (!full)
bs.SetTypeReason(Strings.Interface.FullBecauseBackendIsEmpty);
full = true;
}
else
{
//A prioir backup exists, extract the compression and encryption modules used in the most recent entry
string compression = null;
string encryption = null;
for (int i = backupsets.Count - 1; compression == null && i >= 0; i--)
{
for (int j = backupsets[i].Incrementals.Count - 1; compression == null && j >= 0; j--)
for (int k = backupsets[i].Incrementals[j].Volumes.Count - 1; compression == null && k >= 0; k--)
{
compression = backupsets[i].Incrementals[j].Volumes[k].Key.Compression;
encryption = backupsets[i].Incrementals[j].Volumes[k].Key.EncryptionMode;
if (compression != null)
break;
}
for (int k = backupsets[i].Volumes.Count - 1; compression == null && k >= 0; k--)
{
compression = backupsets[i].Volumes[k].Key.Compression;
encryption = backupsets[i].Volumes[k].Key.EncryptionMode;
if (compression != null)
break;
}
}
if (compression != null)
{
m_options.SetEncryptionModuleDefault(encryption);
m_options.SetCompressionModuleDefault(compression);
}
}
string fullCriteria1 = null;
string fullCriteria2 = null;
if (!full)
{
full = DateTime.Now > m_options.FullIfOlderThan(backupsets[backupsets.Count - 1].Time);
if (full)
bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue));
else if (!string.IsNullOrEmpty(m_options.FullIfOlderThanValue))
fullCriteria1 = string.Format(Strings.Interface.IncrementalBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue);
}
if (!full && m_options.FullIfMoreThanNIncrementals > 0)
{
full = backupsets[backupsets.Count - 1].Incrementals.Count >= m_options.FullIfMoreThanNIncrementals;
if (full)
bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals));
else
fullCriteria2 = string.Format(Strings.Interface.IncrementalBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals);
}
bs.Full = full;
if (!full)
{
if (fullCriteria1 == null && fullCriteria2 == null)
bs.SetTypeReason(Strings.Interface.IncrementalBecauseNoFlagsWereSet);
else if (fullCriteria2 == null)
bs.SetTypeReason(fullCriteria1);
else if (fullCriteria1 == null)
bs.SetTypeReason(fullCriteria2);
else
bs.SetTypeReason(fullCriteria1 + ". " + fullCriteria2);
}
List<string> controlfiles = new List<string>();
if (!string.IsNullOrEmpty(m_options.SignatureControlFiles))
controlfiles.AddRange(m_options.SignatureControlFiles.Split(System.IO.Path.PathSeparator));
int vol = 0;
long totalsize = 0;
Manifestfile manifest = new Manifestfile();
using (Utility.TempFolder tempfolder = new Duplicati.Library.Utility.TempFolder())
{
List<KeyValuePair<ManifestEntry, Library.Interface.ICompression>> patches = new List<KeyValuePair<ManifestEntry, Duplicati.Library.Interface.ICompression>>();
if (!full)
{
m_incrementalFraction = INCREMENAL_COST;
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.Add(backupsets[backupsets.Count - 1]);
entries.AddRange(backupsets[backupsets.Count - 1].Incrementals);
//Check before we start the download
CheckLiveControl();
VerifyBackupChainWithFiles(backend, entries[entries.Count - 1]);
if (m_options.CreateVerificationFile)
verification = new VerificationFile(entries, backend.FilenameStrategy);
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, "");
patches = FindPatches(backend, entries, tempfolder, false, bs);
//Check before we start the download
CheckLiveControl();
Manifestfile latest = GetManifest(backend, backupsets[backupsets.Count - 1]);
//Manifest version 1 does not support multiple folders
if (latest.Version == 1)
latest.SourceDirs = new string[] { sources[0] };
if (latest.SourceDirs.Length != sources.Length)
{
if (m_options.FullIfSourceFolderChanged)
{
Logging.Log.WriteMessage("Source folder count changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information);
if (!full)
bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged);
full = true;
}
else
throw new Exception(string.Format(Strings.Interface.NumberOfSourceFoldersHasChangedError, latest.SourceDirs.Length, sources.Length));
}
else
{
if (!m_options.AllowSourceFolderChange)
{
foreach (string s1 in latest.SourceDirs)
{
bool found = false;
foreach (string s2 in sources)
if (s1.Equals(s2, Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
{
found = true;
break;
}
if (!found)
{
if (m_options.FullIfSourceFolderChanged)
{
Logging.Log.WriteMessage("Source folders changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information);
if (!full)
bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged);
full = true;
break; //Exit the folder loop
}
else
throw new Exception(string.Format(Strings.Interface.SourceFoldersHasChangedError, s1));
}
}
manifest.SourceDirs = latest.SourceDirs;
}
else
{
manifest.SourceDirs = sources;
}
}
}
DateTime backuptime = DateTime.Now;
DateTime backupchaintime;
if (full)
{
patches.Clear();
m_incrementalFraction = 0.0;
manifest.SourceDirs = sources;
if (m_options.CreateVerificationFile)
verification = new VerificationFile(new ManifestEntry[0], backend.FilenameStrategy);
backupchaintime = backuptime;
}
else
{
backupchaintime = patches[0].Key.Time;
manifest.PreviousManifestFilename = patches[patches.Count - 1].Key.Filename;
manifest.PreviousManifestHash = patches[patches.Count - 1].Key.RemoteHash;
}
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusBuildingFilelist, "");
bool completedWithoutChanges;
using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(manifest.SourceDirs, bs, m_options.Filter, patches))
{
CheckLiveControl();
dir.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupRSyncDir_ProgressEvent);
dir.DisableFiletimeCheck = m_options.DisableFiletimeCheck;
dir.MaxFileSize = m_options.SkipFilesLargerThan;
using (new Logging.Timer("Initiating multipass"))
dir.InitiateMultiPassDiff(full, m_options);
string tempVolumeFolder = m_options.AsynchronousUpload ? m_options.AsynchronousUploadFolder : m_options.TempDir;
bool done = false;
while (!done && totalsize < m_options.MaxSize)
{
using (new Logging.Timer("Multipass " + (vol + 1).ToString()))
using (Utility.TempFile signaturefile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString())))
using (Utility.TempFile contentfile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString())))
{
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusCreatingVolume, vol + 1), "");
CheckLiveControl();
using (Library.Interface.ICompression signaturearchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, signaturefile, m_options.RawOptions))
using (Library.Interface.ICompression contentarchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, contentfile, m_options.RawOptions))
{
//If we are all out, stop now, this may cause incomplete partial files
if (m_options.MaxSize - totalsize < (contentarchive.FlushBufferSize + backend.FileSizeOverhead))
break;
//Add signature files to archive
foreach (string s in controlfiles)
if (!string.IsNullOrEmpty(s))
using (System.IO.Stream cs = signaturearchive.CreateFile(System.IO.Path.Combine(RSync.RSyncDir.CONTROL_ROOT, System.IO.Path.GetFileName(s))))
using (System.IO.FileStream fs = System.IO.File.OpenRead(s))
Utility.Utility.CopyStream(fs, cs);
//Only add control files to the very first volume
controlfiles.Clear();
done = dir.MakeMultiPassDiff(signaturearchive, contentarchive, (Math.Min(m_options.VolumeSize, m_options.MaxSize - totalsize)) - backend.FileSizeOverhead);
//TODO: This is not the correct size, we need to account for file size overhead as well
totalsize += signaturearchive.Size;
totalsize += contentarchive.Size;
//TODO: This is not the best way to determine this
if (totalsize >= m_options.MaxSize)
dir.FinalizeMultiPass(signaturearchive, contentarchive, long.MaxValue);
}
completedWithoutChanges = done && !dir.AnyChangesFound;
if (m_options.UploadUnchangedBackups || full)
completedWithoutChanges = false;
if (!completedWithoutChanges)
{
if (m_options.AsynchronousUpload)
{
m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload;
m_allowUploadProgress = true;
m_allowUploadProgressAfter = DateTime.Now.AddSeconds(1);
}
else
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingContentVolume, vol + 1), "");
//Last check before we upload, we do not interrupt transfers
CheckLiveControl();
//The backendwrapper will remove these
signaturefile.Protected = true;
contentfile.Protected = true;
ContentEntry ce = new ContentEntry(backuptime, full, vol + 1);
SignatureEntry se = new SignatureEntry(backuptime, full, vol + 1);
using (new Logging.Timer("Writing delta file " + (vol + 1).ToString()))
backend.Put(ce, contentfile);
if (!m_options.AsynchronousUpload)
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingSignatureVolume, vol + 1), "");
using (new Logging.Timer("Writing remote signatures"))
backend.Put(se, signaturefile);
manifest.AddEntries(ce, se);
if (verification != null)
{
verification.AddFile(ce);
verification.AddFile(se);
}
}
}
if (!completedWithoutChanges)
{
//The backend wrapper will remove these
Utility.TempFile mf = new Duplicati.Library.Utility.TempFile();
using (new Logging.Timer("Writing manifest " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK")))
{
//Alternate primary/secondary
ManifestEntry mfe = new ManifestEntry(backuptime, full, manifest.SignatureHashes.Count % 2 != 0);
manifest.SelfFilename = backend.GenerateFilename(mfe);
manifest.Save(mf);
if (!m_options.AsynchronousUpload)
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingManifestVolume, vol + 1), "");
//Write the file
mf.Protected = true;
backend.Put(mfe, mf);
if (verification != null)
verification.UpdateManifest(mfe);
}
if (verification != null)
{
using (new Logging.Timer("Writing verification " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK")))
{
Utility.TempFile vt = new Duplicati.Library.Utility.TempFile();
verification.Save(vt);
if (!m_options.AsynchronousUpload)
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusUploadingVerificationVolume, "");
vt.Protected = true;
backend.Put(new VerificationEntry(backupchaintime), vt);
}
}
if (m_options.AsynchronousUpload)
m_allowUploadProgress = false;
//The file volume counter
vol++;
}
}
}
//If we are running asynchronous, we now enter the end-game
if (m_options.AsynchronousUpload)
{
m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload;
m_allowUploadProgress = true;
m_allowUploadProgressAfter = DateTime.Now;
//Before we clear the temp folder, we need to ensure that all volumes are uploaded.
//To allow the UI to show some progress while uploading, we perform the remaining
// uploads synchronous
List<KeyValuePair<BackupEntryBase, string>> pendingUploads = backend.ExtractPendingUploads();
//Figure out what volume number we are at
foreach (KeyValuePair<BackupEntryBase, string> p in pendingUploads)
if (p.Key is ManifestEntry)
vol--;
double unitcost = m_asyncReserved / pendingUploads.Count;
//The upload each remaining volume in order
foreach (KeyValuePair<BackupEntryBase, string> p in pendingUploads)
{
string msg;
if (p.Key is ManifestEntry)
{
vol++;
msg = string.Format(Strings.Interface.StatusUploadingManifestVolume, vol);
}
else if (p.Key is SignatureEntry)
msg = string.Format(Strings.Interface.StatusUploadingSignatureVolume, ((SignatureEntry)p.Key).Volumenumber);
else if (p.Key is ContentEntry)
{
msg = string.Format(Strings.Interface.StatusUploadingContentVolume, ((ContentEntry)p.Key).Volumenumber);
//We allow a stop or pause request here
CheckLiveControl();
}
else if (p.Key is VerificationEntry)
msg = Strings.Interface.StatusUploadingVerificationVolume;
else
throw new InvalidOperationException();
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, msg, "");
backend.Put(p.Key, p.Value);
m_asyncReserved -= unitcost;
m_progress += unitcost;
}
}
}
}
catch(Exception ex)
{
//If this is a controlled user-requested stop, wait for the current upload to complete
if (backend != null && ex is LiveControl.ExecutionStoppedException)
{
try
{
if (m_options.AsynchronousUpload)
{
m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload;
m_allowUploadProgress = true;
m_allowUploadProgressAfter = DateTime.Now;
//Wait for the current upload to complete and then delete all remaining temporary files
foreach (KeyValuePair<BackupEntryBase, string> p in backend.ExtractPendingUploads())
try
{
if (System.IO.File.Exists(p.Value))
System.IO.File.Delete(p.Value);
}
catch { } //Better to delete as many as possible rather than choke on a single file
}
}
catch { } //We already have an exception, just go with that
}
if (backend == null || backend.ManifestUploads == 0)
{
Logging.Log.WriteMessage(string.Format(Strings.Interface.ErrorRunningBackup, ex.Message), Logging.LogMessageType.Error);
throw; //This also activates "finally", unlike in other languages...
}
Logging.Log.WriteMessage(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), Logging.LogMessageType.Warning);
bs.LogError(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), ex);
}
finally
{
m_progress = 100.0;
if (backend != null)
try { backend.Dispose(); }
catch { }
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, "");
}
}
bs.EndTime = DateTime.Now;
return bs.ToString();
}
public List<KeyValuePair<BackupEntryBase, Exception>> VerifyBackupChain()
{
CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.Verify);
SetupCommonOptions(stats);
List<KeyValuePair<BackupEntryBase, Exception>> results = new List<KeyValuePair<BackupEntryBase, Exception>>();
if (m_options.DontReadManifests)
throw new InvalidOperationException(Strings.Interface.ManifestsMustBeRead);
if (m_options.SkipFileHashChecks)
throw new InvalidOperationException(Strings.Interface.CannotVerifyWithoutHashes);
if (!string.IsNullOrEmpty(m_options.SignatureCachePath))
{
stats.LogWarning(Strings.Interface.DisablingSignatureCacheForVerification, null);
m_options.SignatureCachePath = null;
}
using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options))
{
//Find the spot in the chain where we start
ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime);
//Get the list of manifests to validate
List<ManifestEntry> entries = new List<ManifestEntry>();
entries.Add(bestFit);
entries.AddRange(bestFit.Incrementals);
entries.Reverse();
foreach (ManifestEntry me in entries)
{
Manifestfile mf = null;
try
{
mf = GetManifest(backend, me);
VerifyBackupChainWithFiles(backend, me);
if (mf.SignatureHashes.Count != me.Volumes.Count)
results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, new Exception(string.Format(Strings.Interface.ManifestAndFileCountMismatchError, mf.SelfFilename, mf.SignatureHashes.Count, me.Volumes.Count))));
else
results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, null));
}
catch (Exception ex)
{
results.Add(new KeyValuePair<BackupEntryBase,Exception>(me, ex));
}
if (mf != null)
{
int volumes = Math.Min(mf.SignatureHashes.Count, me.Volumes.Count);
for(int i = 0; i <volumes; i++)
{
if (m_options.Verificationlevel == VerificationLevel.Signature || m_options.Verificationlevel == VerificationLevel.Full)
{
try
{
using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
backend.Get(me.Volumes[i].Key, mf, tf, mf.SignatureHashes[i]);
results.Add(new KeyValuePair<BackupEntryBase, Exception>(me.Volumes[i].Key, null));
}
catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me.Volumes[i].Key, ex)); }
}
if (m_options.Verificationlevel == VerificationLevel.Full)
{
try
{
using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
backend.Get(me.Volumes[i].Value, mf, tf, mf.ContentHashes[i]);
results.Add(new KeyValuePair<BackupEntryBase, Exception>(me.Volumes[i].Value, null));
}
catch (Exception ex) { results.Add(new KeyValuePair<BackupEntryBase,Exception>(me.Volumes[i].Value, ex)); }
}
}
}
}
//Re-generate verification file
if (m_options.CreateVerificationFile)
{
//Stop any async operations
if (m_options.AsynchronousUpload)
backend.ExtractPendingUploads();
VerificationFile vf = new VerificationFile(entries, new FilenameStrategy(m_options));
using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile())
{
vf.Save(tf);
tf.Protected = true;
backend.Put(new VerificationEntry(entries[entries.Count - 1].Time), tf);
}
}
}
return results;
}
