public void Run(string[] sources, Library.Utility.IFilter filter)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
using(m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;
Utility.UpdateOptionsFromDb(m_database, m_options);
Utility.VerifyParameters(m_database, m_options);
if (m_database.RepairInProgress)
throw new Exception("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.");
m_blocksize = m_options.Blocksize;
m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
m_blocklistbuffer = new byte[m_options.Blocksize];
m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (m_blockhasher == null)
throw new Exception(Strings.Common.InvalidHashAlgorithm(m_options.BlockHashAlgorithm));
if (m_filehasher == null)
throw new Exception(Strings.Common.InvalidHashAlgorithm(m_options.FileHashAlgorithm));
if (!m_blockhasher.CanReuseTransform)
throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.BlockHashAlgorithm));
if (!m_filehasher.CanReuseTransform)
throw new Exception(Strings.Common.InvalidCryptoSystem(m_options.FileHashAlgorithm));
m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
System.Threading.Thread parallelScanner = null;
try
{
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using(var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
{
using(var snapshot = GetSnapshot(sources, m_options, m_result))
{
// Start parallel scan
if (m_options.ChangedFilelist == null || m_options.ChangedFilelist.Length < 1)
{
parallelScanner = new System.Threading.Thread(CountFilesThread) {
Name = "Read ahead file counter",
IsBackground = true
};
parallelScanner.Start(snapshot);
}
PreBackupVerify(backend);
// Verify before uploading a synthetic list
m_database.VerifyConsistency(null, m_options.Blocksize, m_options.BlockhashSize);
UploadSyntheticFilelist(backend);
m_database.BuildLookupTable(m_options);
m_transaction = m_database.BeginTransaction();
var repcnt = 0;
while(repcnt < 100 && m_database.GetRemoteVolumeID(filesetvolume.RemoteFilename) >= 0)
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
if (m_database.GetRemoteVolumeID(filesetvolume.RemoteFilename) >= 0)
throw new Exception("Unable to generate a unique fileset name");
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
var filesetvolumeid = m_database.RegisterRemoteVolume(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
m_database.CreateFileset(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time, m_transaction);
RunMainOperation(snapshot, backend);
//If the scanner is still running for some reason, make sure we kill it now
if (parallelScanner != null && parallelScanner.IsAlive)
parallelScanner.Abort();
}
var lastVolumeSize = FinalizeRemoteVolumes(backend);
using(new Logging.Timer("UpdateChangeStatistics"))
m_database.UpdateChangeStatistics(m_result);
using(new Logging.Timer("VerifyConsistency"))
m_database.VerifyConsistency(m_transaction, m_options.Blocksize, m_options.BlockhashSize);
UploadRealFileList(backend, filesetvolume);
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
using(new Logging.Timer("Async backend wait"))
backend.WaitForComplete(m_database, m_transaction);
if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
CompactIfRequired(backend, lastVolumeSize);
if (m_options.UploadVerificationFile)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
FilelistProcessor.UploadVerificationFile(backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
}
if (m_options.Dryrun)
{
m_transaction.Rollback();
m_transaction = null;
}
else
{
using(new Logging.Timer("CommitFinalizingBackup"))
m_transaction.Commit();
m_transaction = null;
m_database.Vacuum();
if (m_result.TaskControlRendevouz() != TaskControlState.Stop && !m_options.NoBackendverification)
{
PostBackupVerification();
}
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
m_database.WriteResults();
m_database.PurgeLogData(m_options.LogRetention);
return;
}
}
catch (Exception ex)
{
m_result.AddError("Fatal error", ex);
throw;
}
finally
{
if (parallelScanner != null && parallelScanner.IsAlive)
{
parallelScanner.Abort();
parallelScanner.Join(500);
if (parallelScanner.IsAlive)
m_result.AddWarning("Failed to terminate filecounter thread", null);
}
if (m_transaction != null)
try { m_transaction.Rollback(); }
catch (Exception ex) { m_result.AddError(string.Format("Rollback error: {0}", ex.Message), ex); }
}
}
}
public static int Run(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 2 && args.Count != 3)
{
Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
return(100);
}
var folder = Path.GetFullPath(args[1]);
if (!Directory.Exists(folder))
{
Console.WriteLine("Folder not found: {0}", folder);
return(100);
}
Directory.SetCurrentDirectory(folder);
if (args.Count == 2)
{
var times = ParseListFiles(folder);
foreach (var v in times.Zip(Enumerable.Range(0, times.Length), (a, b) => new KeyValuePair <int, DateTime>(b, a.Key)))
{
Console.WriteLine("{0}: {1}", v.Key, v.Value.ToLocalTime());
}
}
else
{
var file = SelectListFile(args[2], folder);
var p = Library.Main.Volumes.VolumeBase.ParseFilename(file);
Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(p.CompressionModule, file, new Duplicati.Library.Main.Options(options));
foreach (var f in EnumerateFilesInDList(file, filter, options))
{
Console.WriteLine("{0} ({1})", f.Path, Library.Utility.Utility.FormatSizeString(f.Size));
}
}
return(0);
}
public Duplicati.Library.Interface.IRepairResults Repair(Library.Utility.IFilter filter = null)
{
return(RunAction(new RepairResults(), ref filter, (result) => {
new Operation.RepairHandler(m_backend, m_options, result).Run(filter);
}));
}
public Duplicati.Library.Interface.IListResults List(string filterstring, Library.Utility.IFilter filter = null)
{
return(List(filterstring == null ? null : new string[] { filterstring }, null));
}
private static Library.Utility.IFilter ApplyFilter(Serialization.Interface.IBackup backup, Library.Utility.IFilter filter)
{
var f2 = backup.Filters;
if (f2 != null && f2.Length > 0)
{
var nf =
(from n in f2
let exp =
n.Expression.StartsWith("[", StringComparison.Ordinal) && n.Expression.EndsWith("]", StringComparison.Ordinal)
? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression)
: SpecialFolders.ExpandEnvironmentVariables(n.Expression)
orderby n.Order
select(Library.Utility.IFilter)(new Library.Utility.FilterExpression(exp, n.Include)))
.Aggregate((a, b) => Library.Utility.FilterExpression.Combine(a, b));
filter = Library.Utility.FilterExpression.Combine(filter, nf);
}
return(filter);
}
private async Task RunAsync(string[] sources, Library.Utility.IFilter filter)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
// New isolated scope for each operation
using (new IsolatedChannelScope())
using (m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;
// Check the database integrity
Utility.UpdateOptionsFromDb(m_database, m_options);
Utility.VerifyParameters(m_database, m_options);
var probe_path = m_database.GetFirstPath();
if (probe_path != null && Duplicati.Library.Utility.Utility.GuessDirSeparator(probe_path) != System.IO.Path.DirectorySeparatorChar.ToString())
{
throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported");
}
if (m_database.PartiallyRecreated)
{
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
}
if (m_database.RepairInProgress)
{
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress");
}
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
Task parallelScanner = null;
Task uploader = null;
try
{
// Setup runners and instances here
using (var db = new Backup.BackupDatabase(m_database, m_options))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
using (var stats = new Backup.BackupStatsCollector(m_result))
using (var bk = new Common.BackendHandler(m_options, m_backendurl, db, stats, m_result.TaskReader))
// Keep a reference to these channels to avoid shutdown
using (var uploadtarget = ChannelManager.GetChannel(Backup.Channels.BackendRequest.ForWrite))
{
long filesetid;
var counterToken = new CancellationTokenSource();
using (var snapshot = GetSnapshot(sources, m_options))
{
try
{
// Start parallel scan, or use the database
if (m_options.DisableFileScanner)
{
var d = m_database.GetLastBackupFileCountAndSize();
m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);
}
else
{
parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, m_result, m_options, m_sourceFilter, m_filter, m_result.TaskReader, counterToken.Token);
}
// Make sure the database is sane
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true);
// Start the uploader process
uploader = Backup.BackendUploader.Run(bk, m_options, db, m_result, m_result.TaskReader, stats);
// If we have an interrupted backup, grab the
string lasttempfilelist = null;
long lasttempfileid = -1;
if (!m_options.DisableSyntheticFilelist)
{
var candidates = (await db.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToArray();
if (candidates.Length > 0)
{
lasttempfileid = candidates.Last().Key;
lasttempfilelist = m_database.GetRemoteVolumeFromID(lasttempfileid).Name;
}
}
// TODO: Rewrite to using the uploader process, or the BackendHandler interface
// Do a remote verification, unless disabled
PreBackupVerify(backend, lasttempfilelist);
// If the previous backup was interrupted, send a synthetic list
await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskReader, lasttempfilelist, lasttempfileid);
// Grab the previous backup ID, if any
var prevfileset = m_database.FilesetTimes.FirstOrDefault();
if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime())
{
throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()));
}
var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
// Rebuild any index files that are missing
await Backup.RecreateMissingIndexFiles.Run(db, m_options, m_result, m_result.TaskReader);
// This should be removed as the lookups are no longer used
m_database.BuildLookupTable(m_options);
// Prepare the operation by registering the filelist
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
var repcnt = 0;
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
{
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
}
if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
{
throw new Exception("Unable to generate a unique fileset name");
}
var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time);
// create USN-based scanner if enabled
var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid);
// Run the backup operation
if (await m_result.TaskReader.ProgressAsync)
{
await RunMainOperation(sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, lastfilesetid);
}
}
finally
{
//If the scanner is still running for some reason, make sure we kill it now
counterToken.Cancel();
}
}
// Ensure the database is in a sane state after adding data
using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency"))
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false);
// Send the actual filelist
if (await m_result.TaskReader.ProgressAsync)
{
await Backup.UploadRealFilelist.Run(m_result, db, m_options, filesetvolume, filesetid, m_result.TaskReader);
}
// Wait for upload completion
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploader);
// Make sure we have the database up-to-date
await db.CommitTransactionAsync("CommitAfterUpload", false);
// TODO: Remove this later
m_transaction = m_database.BeginTransaction();
if (await m_result.TaskReader.ProgressAsync)
{
CompactIfRequired(backend, lastVolumeSize);
}
if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
FilelistProcessor.UploadVerificationFile(backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
}
if (m_options.Dryrun)
{
m_transaction.Rollback();
m_transaction = null;
}
else
{
using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup"))
m_transaction.Commit();
m_transaction = null;
if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
{
if (m_options.NoBackendverification)
{
UpdateStorageStatsFromDatabase();
}
else
{
PostBackupVerification();
}
}
}
m_database.WriteResults();
m_database.PurgeLogData(m_options.LogRetention);
if (m_options.AutoVacuum)
{
m_database.Vacuum();
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
return;
}
}
catch (Exception ex)
{
var aex = BuildException(ex, uploader, parallelScanner);
Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error");
if (aex == ex)
{
throw;
}
throw aex;
}
finally
{
if (parallelScanner != null && !parallelScanner.IsCompleted)
{
parallelScanner.Wait(500);
}
// TODO: We want to commit? always?
if (m_transaction != null)
{
try { m_transaction.Rollback(); }
catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); }
}
}
}
}
public static int SendMail(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args != null && args.Count != 0)
{
Console.WriteLine("Command takes no arguments");
return(200);
}
using (var i = new Library.Main.Controller("dummy://", options, new ConsoleOutput(options)))
{
foreach (var l in i.SendMail().Lines)
{
Console.WriteLine(l);
}
}
return(0);
}
public static int CreateBugReport(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
// Support for not adding the --auth-username if possible
string dbpath = null;
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
if (args.Count > 0)
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(args[0], new Duplicati.Library.Main.Options(options), false, true);
}
if (dbpath == null)
{
Console.WriteLine("No local database found, please add --{0}", "dbpath");
return(100);
}
else
{
options["dbpath"] = dbpath;
}
}
if (args.Count == 0)
{
args = new List <string>(new string[] { "file://unused", "report" });
}
else if (args.Count == 1)
{
args.Add("report");
}
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
i.CreateLogDatabase(args[1]);
Console.WriteLine("Completed!");
Console.WriteLine();
Console.WriteLine("Please examine the log table of the database to see that no filenames are accidentially left over.");
Console.WriteLine("If you are concerned that your filenames may contain sensitive information,");
Console.WriteLine(" do not attach the database to an issue!!!");
Console.WriteLine();
return(0);
}
public static int ListBrokenFiles(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 1)
{
return(PrintWrongNumberOfArguments(args, 1));
}
var con = new ConsoleOutput(options);
var previd = -1L;
var outputcount = 0L;
var verbose = Duplicati.Library.Utility.Utility.ParseBoolOption(options, "verbose");
using (var i = new Library.Main.Controller(args[0], options, con))
i.ListBrokenFiles(filter, (id, time, count, path, size) =>
{
if (previd != id)
{
previd = id;
outputcount = 0;
con.MessageEvent(string.Format("{0}\t: {1}\t({2} match(es))", id, time.ToLocalTime(), count));
}
con.MessageEvent(string.Format("\t{0} ({1})", path, Library.Utility.Utility.FormatSizeString(size)));
outputcount++;
if (outputcount >= 5 && !verbose && count != outputcount)
{
con.MessageEvent(string.Format("\t ... and {0} more, (use --{1} to list all)", count - outputcount, "verbose"));
return(false);
}
return(true);
});
return(0);
}
public static int PurgeBrokenFiles(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 1)
{
return(PrintWrongNumberOfArguments(args, 1));
}
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
{
var res = i.PurgeBrokenFiles(filter);
}
return(0);
}
public static int PurgeFiles(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count < 1)
{
return(PrintWrongNumberOfArguments(args, 1));
}
var backend = args[0];
var paths = args.Skip(1).ToArray();
if (paths.Length > 0)
{
if (filter == null || filter.Empty)
{
filter = new Library.Utility.FilterExpression(paths);
}
else
{
Console.WriteLine("You cannot combine filters and paths on the commandline");
return(200);
}
}
else if (filter == null || filter.Empty)
{
Console.WriteLine("You must provide either filename filters, or a list of paths to remove");
return(200);
}
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
i.PurgeFiles(filter);
return(0);
}
public static int SystemInfo(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args != null && args.Count != 0)
{
Console.WriteLine("Command takes no arguments");
return(200);
}
using (var i = new Library.Main.Controller("dummy://", options, new ConsoleOutput(options)))
foreach (var line in i.SystemInfo().Lines)
{
Console.WriteLine(line);
}
Console.WriteLine("Know locales: {0}", string.Join(", ", Library.Localization.LocalizationService.AllLocales));
Console.WriteLine("Translated locales: {0}", string.Join(", ", Library.Localization.LocalizationService.SupportedCultures));
return(0);
}
public static int TestFilters(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args == null || args.Count < 1)
{
Console.WriteLine("No source paths given");
return(200);
}
options["verbose"] = "true";
using (var i = new Library.Main.Controller("dummy://", options, new ConsoleOutput(options)))
{
var result = i.TestFilter(args.ToArray(), filter);
Console.WriteLine("Matched {0} files ({1})", result.FileCount, Duplicati.Library.Utility.Utility.FormatSizeString(result.FileSize));
}
return(0);
}
public static int Compact(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 1)
{
return(PrintWrongNumberOfArguments(args, 1));
}
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
i.Compact();
return(0);
}
public static int Examples(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
Duplicati.CommandLine.Help.PrintUsage("example", options);
return(0);
}
public static int Test(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 1 && args.Count != 2)
{
return(PrintWrongNumberOfArguments(args, 1));
}
var tests = 1L;
if (args.Count == 2)
{
if (new string[] { "all", "everything" }.Contains(args[1], StringComparer.InvariantCultureIgnoreCase))
{
tests = long.MaxValue;
}
else
{
tests = Convert.ToInt64(args[1]);
}
}
Library.Interface.ITestResults result;
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
result = i.Test(tests);
var totalFiles = result.Verifications.Count();
if (totalFiles == 0)
{
Console.WriteLine("No files examined, is the remote destination is empty?");
}
else
{
var filtered = from n in result.Verifications where n.Value.Count() != 0 select n;
if (filtered.Count() == 0)
{
Console.WriteLine("Examined {0} files and found no errors", totalFiles);
}
else
{
if (Library.Utility.Utility.ParseBoolOption(options, "verbose"))
{
foreach (var n in result.Verifications)
{
var changecount = n.Value.Count();
if (changecount == 0)
{
Console.WriteLine("{0}: No errors", n.Key);
}
else
{
Console.WriteLine("{0}: {1} errors", n.Key, changecount);
foreach (var c in n.Value)
{
Console.WriteLine("{0}: {1}", c.Key, c.Value);
}
Console.WriteLine();
}
}
}
else
{
Console.WriteLine("Examined {0} files and found errors in the following files:", totalFiles);
foreach (var n in filtered)
{
Console.WriteLine(n.Key);
}
Console.WriteLine();
}
}
}
return(0);
}
public static int Help(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count < 1)
{
Duplicati.CommandLine.Help.PrintUsage("help", options);
}
else
{
Duplicati.CommandLine.Help.PrintUsage(args[0], options);
}
return(0);
}
public static int ListChanges(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count < 1)
{
return(PrintWrongNumberOfArguments(args, 1));
}
// Support for not adding the --auth-username if possible
string dbpath;
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(args[0], new Duplicati.Library.Main.Options(options), false, true);
if (dbpath != null)
{
options["dbpath"] = dbpath;
}
}
// Don't ask for passphrase if we have a local db
if (!string.IsNullOrEmpty(dbpath) && System.IO.File.Exists(dbpath) && !options.ContainsKey("no-encryption") && !Duplicati.Library.Utility.Utility.ParseBoolOption(options, "no-local-db"))
{
string passphrase;
options.TryGetValue("passphrase", out passphrase);
if (string.IsNullOrEmpty(passphrase))
{
options["no-encryption"] = "true";
}
}
Library.Interface.IListChangesResults result;
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
if (args.Count == 2)
{
result = i.ListChanges(null, args[1], null, filter);
}
else
{
result = i.ListChanges(args.Count > 1 ? args[1] : null, args.Count > 2 ? args[2] : null, null, filter);
}
Console.WriteLine("Listing changes");
Console.WriteLine(" {0}: {1}", result.BaseVersionIndex, result.BaseVersionTimestamp);
Console.WriteLine(" {0}: {1}", result.CompareVersionIndex, result.CompareVersionTimestamp);
Console.WriteLine();
Console.WriteLine("Size of backup {0}: {1}", result.BaseVersionIndex, Library.Utility.Utility.FormatSizeString(result.PreviousSize));
if (result.ChangeDetails != null)
{
Console.WriteLine();
var added = result.ChangeDetails.Where(x => x.Item1 == Library.Interface.ListChangesChangeType.Added);
var deleted = result.ChangeDetails.Where(x => x.Item1 == Library.Interface.ListChangesChangeType.Deleted);
var modified = result.ChangeDetails.Where(x => x.Item1 == Library.Interface.ListChangesChangeType.Modified);
var count = added.Count();
if (count > 0)
{
Console.WriteLine(" {0} added entries:", count);
foreach (var n in added)
{
Console.WriteLine(" + {0}", n.Item3);
}
Console.WriteLine();
}
count = modified.Count();
if (count > 0)
{
Console.WriteLine(" {0} modified entries:", count);
foreach (var n in modified)
{
Console.WriteLine(" ~ {0}", n.Item3);
}
Console.WriteLine();
}
count = deleted.Count();
if (count > 0)
{
Console.WriteLine("{0} deleted entries:", count);
foreach (var n in deleted)
{
Console.WriteLine(" - {0}", n.Item3);
}
Console.WriteLine();
}
Console.WriteLine();
}
else
{
if (result.AddedFolders > 0)
{
Console.WriteLine(" Added folders: {0}", result.AddedFolders);
}
if (result.AddedSymlinks > 0)
{
Console.WriteLine(" Added symlinks: {0}", result.AddedSymlinks);
}
if (result.AddedFiles > 0)
{
Console.WriteLine(" Added files: {0}", result.AddedFiles);
}
if (result.DeletedFolders > 0)
{
Console.WriteLine(" Deleted folders: {0}", result.DeletedFolders);
}
if (result.DeletedSymlinks > 0)
{
Console.WriteLine(" Deleted symlinks: {0}", result.DeletedSymlinks);
}
if (result.DeletedFiles > 0)
{
Console.WriteLine(" Deleted files: {0}", result.DeletedFiles);
}
if (result.ModifiedFolders > 0)
{
Console.WriteLine(" Modified folders: {0}", result.ModifiedFolders);
}
if (result.ModifiedSymlinks > 0)
{
Console.WriteLine(" Modified symlinka: {0}", result.ModifiedSymlinks);
}
if (result.ModifiedFiles > 0)
{
Console.WriteLine(" Modified files: {0}", result.ModifiedFiles);
}
if (result.AddedFolders + result.AddedSymlinks + result.AddedFolders +
result.ModifiedFolders + result.ModifiedSymlinks + result.ModifiedFiles +
result.DeletedFolders + result.DeletedSymlinks + result.DeletedFiles == 0)
{
Console.WriteLine(" No changes found");
}
}
Console.WriteLine("Size of backup {0}: {1}", result.CompareVersionIndex, Library.Utility.Utility.FormatSizeString(result.CurrentSize));
return(0);
}
public static int Affected(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
var backend = args[0];
args.RemoveAt(0);
if (args.Count == 0)
{
Console.WriteLine("You must specify at least a remote filename");
return(200);
}
// Support for not adding the --auth-username if possible
string dbpath;
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(backend, new Duplicati.Library.Main.Options(options), false, true);
if (dbpath != null)
{
options["dbpath"] = dbpath;
}
}
// Don't ask for passphrase if we have a local db
if (!string.IsNullOrEmpty(dbpath) && System.IO.File.Exists(dbpath) && !options.ContainsKey("no-encryption") && !Duplicati.Library.Utility.Utility.ParseBoolOption(options, "no-local-db"))
{
string passphrase;
options.TryGetValue("passphrase", out passphrase);
if (string.IsNullOrEmpty(passphrase))
{
options["no-encryption"] = "true";
}
}
using (var i = new Library.Main.Controller(backend, options, new ConsoleOutput(options)))
{
var res = i.ListAffected(args);
if (res.Filesets != null && res.Filesets.Count() != 0)
{
Console.WriteLine("The following filesets are affected:");
foreach (var e in res.Filesets)
{
Console.WriteLine("{0}\t: {1}", e.Version, e.Time);
}
Console.WriteLine();
}
if (res.RemoteVolumes != null && res.RemoteVolumes.Count() != 0)
{
}
var filecount = res.Files.Count();
if (filecount == 0 || (filecount > 10 && !Duplicati.Library.Utility.Utility.ParseBoolOption(options, "verbose")))
{
Console.WriteLine("A total of {0} file(s) are affected (use --{1} to see filenames)", res.Files.Count(), "verbose");
}
else
{
Console.WriteLine("The following files are affected:");
foreach (var file in res.Files)
{
Console.WriteLine(file.Path);
}
}
Console.WriteLine();
var logcount = res.LogMessages.Count();
if (logcount == 0 || (logcount > 10 && !Duplicati.Library.Utility.Utility.ParseBoolOption(options, "verbose")))
{
Console.WriteLine("Found {0} related log messages (use --{1} to see the data)", res.Files.Count(), "verbose");
}
else
{
Console.WriteLine("The following related log messages were found:");
foreach (var log in res.LogMessages)
{
if (log.Message.Length > 100 && !Duplicati.Library.Utility.Utility.ParseBoolOption(options, "verbose"))
{
Console.WriteLine("{0}: {1}", log.Timestamp, log.Message.Substring(0, 96) + " ...");
}
else
{
Console.WriteLine("{0}: {1}", log.Timestamp, log.Message);
}
}
}
Console.WriteLine();
return(0);
}
}
public void Run(string[] sources, Library.Utility.IFilter filter)
{
RunAsync(sources, filter).WaitForTaskOrThrow();
}
public static int List(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
filter = filter ?? new Duplicati.Library.Utility.FilterExpression();
if (Duplicati.Library.Utility.Utility.ParseBoolOption(options, "list-sets-only"))
{
filter = new Duplicati.Library.Utility.FilterExpression();
}
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
{
var backend = args[0];
args.RemoveAt(0);
if (args.Count == 1)
{
long v;
if (long.TryParse(args[0], out v))
{
if (!options.ContainsKey("version"))
{
args.RemoveAt(0);
args.Add("*");
options["version"] = v.ToString();
}
}
else if (args[0].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[0].StartsWith("["))
{
try
{
var t = Library.Utility.Timeparser.ParseTimeInterval(args[0], DateTime.Now, true);
args.RemoveAt(0);
args.Add("*");
options["time"] = t.ToString();
}
catch
{
}
}
}
// Prefix all filenames with "*/" so we search all folders
for (var ix = 0; ix < args.Count; ix++)
{
if (args[ix].IndexOfAny(new char[] { '*', '?', System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar }) < 0 && !args[ix].StartsWith("["))
{
args[ix] = "*" + System.IO.Path.DirectorySeparatorChar.ToString() + args[ix];
}
}
// Support for not adding the --auth-username if possible
string dbpath;
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(backend, new Duplicati.Library.Main.Options(options), false, true);
if (dbpath != null)
{
options["dbpath"] = dbpath;
}
}
// Don't ask for passphrase if we have a local db
if (!string.IsNullOrEmpty(dbpath) && System.IO.File.Exists(dbpath) && !options.ContainsKey("no-encryption") && !Duplicati.Library.Utility.Utility.ParseBoolOption(options, "no-local-db"))
{
string passphrase;
options.TryGetValue("passphrase", out passphrase);
if (string.IsNullOrEmpty(passphrase))
{
string existing;
options.TryGetValue("disable-module", out existing);
if (string.IsNullOrWhiteSpace(existing))
{
options["disable-module"] = "console-password-input";
}
else
{
options["disable-module"] = string.Join(",", new string[] { existing, "console-password-input" });
}
}
}
bool controlFiles = Library.Utility.Utility.ParseBoolOption(options, "control-files");
options.Remove("control-files");
var res = controlFiles ? i.ListControlFiles(args, filter) : i.List(args, filter);
//If there are no files matching, and we are looking for one or more files,
// try again with all-versions set
var compareFilter = Library.Utility.JoinedFilterExpression.Join(new Library.Utility.FilterExpression(args), filter);
var isRequestForFiles =
!controlFiles && res.Filesets.Count() != 0 &&
(res.Files == null || res.Files.Count() == 0) &&
!compareFilter.Empty;
if (isRequestForFiles && !Library.Utility.Utility.ParseBoolOption(options, "all-versions"))
{
Console.WriteLine("No files matching, looking in all versions");
options["all-versions"] = "true";
options.Remove("time");
options.Remove("version");
res = i.List(args, filter);
}
if (res.Filesets.Count() != 0 && (res.Files == null || res.Files.Count() == 0) && compareFilter.Empty)
{
Console.WriteLine("Listing filesets:");
foreach (var e in res.Filesets)
{
if (e.FileCount >= 0)
{
Console.WriteLine("{0}\t: {1} ({2} files, {3})", e.Version, e.Time, e.FileCount, Library.Utility.Utility.FormatSizeString(e.FileSizes));
}
else
{
Console.WriteLine("{0}\t: {1}", e.Version, e.Time);
}
}
}
else if (isRequestForFiles)
{
Console.WriteLine("No files matched expression");
}
else
{
if (res.Filesets.Count() == 0)
{
Console.WriteLine("No time or version matched a fileset");
}
else if (res.Files == null || res.Files.Count() == 0)
{
Console.WriteLine("Found {0} filesets, but no files matched", res.Filesets.Count());
}
else if (res.Filesets.Count() == 1)
{
var f = res.Filesets.First();
Console.WriteLine("Listing contents {0} ({1}):", f.Version, f.Time);
foreach (var e in res.Files)
{
Console.WriteLine("{0} {1}", e.Path, e.Path.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()) ? "" : "(" + Library.Utility.Utility.FormatSizeString(e.Sizes.First()) + ")");
}
}
else
{
Console.WriteLine("Listing files and versions:");
foreach (var e in res.Files)
{
Console.WriteLine(e.Path);
foreach (var nx in res.Filesets.Zip(e.Sizes, (a, b) => new { Index = a.Version, Time = a.Time, Size = b }))
{
Console.WriteLine("{0}\t: {1} {2}", nx.Index, nx.Time, nx.Size < 0 ? " - " : Library.Utility.Utility.FormatSizeString(nx.Size));
}
Console.WriteLine();
}
}
}
}
return(0);
}
public Duplicati.Library.Interface.IRestoreControlFilesResults RestoreControlFiles(IEnumerable <string> files = null, Library.Utility.IFilter filter = null)
{
return(RunAction(new RestoreControlFilesResults(), ref filter, (result) => {
new Operation.RestoreControlFilesHandler(m_backend, m_options, result).Run(files, filter);
}));
}
public static int Delete(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
var requiredOptions = new string[] { "keep-time", "keep-versions", "version" };
if (!options.Keys.Where(x => requiredOptions.Contains(x, StringComparer.InvariantCultureIgnoreCase)).Any())
{
Console.WriteLine(Strings.Program.DeleteCommandNeedsOptions("delete", requiredOptions));
return(200);
}
using (var i = new Library.Main.Controller(args[0], options, new ConsoleOutput(options)))
{
args.RemoveAt(0);
var res = i.Delete();
if (res.DeletedSets.Count() == 0)
{
Console.WriteLine(Strings.Program.NoFilesetsMatching);
}
else
{
if (res.Dryrun)
{
Console.WriteLine(Strings.Program.WouldDeleteBackups);
}
else
{
Console.WriteLine(Strings.Program.DeletedBackups);
}
foreach (var f in res.DeletedSets)
{
Console.WriteLine(string.Format("{0}: {1}", f.Item1, f.Item2));
}
}
}
return(0);
}
public Duplicati.Library.Interface.IListResults List(Library.Utility.IFilter filter = null)
{
return(List((IEnumerable <string>)null, filter));
}
public static int Restore(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count < 1)
{
return(PrintWrongNumberOfArguments(args, 1));
}
string backend = args[0];
args.RemoveAt(0);
bool controlFiles = Library.Utility.Utility.ParseBoolOption(options, "control-files");
options.Remove("control-files");
// Prefix all filenames with "*/" so we search all folders
for (var ix = 0; ix < args.Count; ix++)
{
if (args[ix].IndexOfAny(new char[] { '*', '?', System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar }) < 0 && !args[ix].StartsWith("["))
{
args[ix] = "*" + System.IO.Path.DirectorySeparatorChar.ToString() + args[ix];
}
}
// suffix all folders with "*" so we restore all contents in the folder
for (var ix = 0; ix < args.Count; ix++)
{
if (args[ix].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[ix].StartsWith("[") && args[ix].EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
{
args[ix] += "*";
}
}
var output = new ConsoleOutput(options);
output.MessageEvent(string.Format("Restore started at {0}", DateTime.Now));
using (var i = new Library.Main.Controller(backend, options, output))
if (controlFiles)
{
var res = i.RestoreControlFiles(args.ToArray(), filter);
output.MessageEvent("Restore control files completed:");
foreach (var s in res.Files)
{
Console.WriteLine(s);
}
}
else
{
using (var periodicOutput = new PeriodicOutput(output, TimeSpan.FromSeconds(5)))
{
output.PhaseChanged += (phase, previousPhase) => {
if (phase == Duplicati.Library.Main.OperationPhase.Restore_PreRestoreVerify)
{
output.MessageEvent("Checking remote backup ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Restore_ScanForExistingFiles)
{
output.MessageEvent("Checking existing target files ...");
}
/*else if (phase == Duplicati.Library.Main.OperationPhase.Restore_DownloadingRemoteFiles)
* output.MessageEvent("Downloading remote files ..."); */
else if (phase == Duplicati.Library.Main.OperationPhase.Restore_PatchWithLocalBlocks)
{
output.MessageEvent("Updating target files with local data ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Restore_PostRestoreVerify)
{
periodicOutput.SetFinished();
periodicOutput.Join(TimeSpan.FromMilliseconds(100));
output.MessageEvent("Verifying restored files ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Restore_ScanForLocalBlocks)
{
output.MessageEvent("Scanning local files for needed data ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Restore_CreateTargetFolders)
{
periodicOutput.SetReady();
}
};
periodicOutput.WriteOutput += (progress, files, size, counting) => {
output.MessageEvent(string.Format(" {0} files need to be restored ({1})", files, Library.Utility.Utility.FormatSizeString(size)));
};
var res = i.Restore(args.ToArray(), filter);
string restorePath;
options.TryGetValue("restore-path", out restorePath);
output.MessageEvent(string.Format("Restored {0} ({1}) files to {2}", res.FilesRestored, Library.Utility.Utility.FormatSizeString(res.SizeOfRestoredFiles), string.IsNullOrEmpty(restorePath) ? "original path" : restorePath));
output.MessageEvent(string.Format("Duration of restore: {0:hh\\:mm\\:ss}", res.Duration));
if (res.FilesRestored > 0 && !Library.Main.Utility.SuppressDonationMessages)
{
output.MessageEvent("***********************************************");
output.MessageEvent("Did we help save your files? If so, please support Duplicati with a donation. We suggest 10€ for private use and 100€ for commercial use.");
output.MessageEvent("");
output.MessageEvent("Paypal: http://goo.gl/P4XJ6S");
output.MessageEvent("Bitcoin: 1L74qa1n5SFKwwyHhECTHBJgcf6WT2rJKf");
output.MessageEvent("***********************************************");
}
if (output.VerboseOutput)
{
Library.Utility.Utility.PrintSerializeObject(res, Console.Out);
}
if (res.Warnings.Count() > 0)
{
return(2);
}
}
}
return(0);
}
public Duplicati.Library.Interface.IListResults List(IEnumerable <string> filterstrings, Library.Utility.IFilter filter = null)
{
return(RunAction(new ListResults(), ref filter, (result) => {
new Operation.ListFilesHandler(m_backend, m_options, result).Run(filterstrings, filter);
}));
}
public static int Backup(List <string> args, Dictionary <string, string> options, Library.Utility.IFilter filter)
{
if (args.Count < 2)
{
return(PrintWrongNumberOfArguments(args, 2));
}
var backend = args[0];
args.RemoveAt(0);
var dirs = args.ToArray();
var output = new ConsoleOutput(options);
Library.Interface.IBackupResults result;
using (var periodicOutput = new PeriodicOutput(output, TimeSpan.FromSeconds(5)))
{
output.MessageEvent(string.Format("Backup started at {0}", DateTime.Now));
output.PhaseChanged += (phase, previousPhase) =>
{
if (previousPhase == Duplicati.Library.Main.OperationPhase.Backup_PostBackupTest)
{
output.MessageEvent("Remote backup verification completed");
}
if (phase == Duplicati.Library.Main.OperationPhase.Backup_ProcessingFiles)
{
output.MessageEvent("Scanning local files ...");
periodicOutput.SetReady();
}
else if (phase == Duplicati.Library.Main.OperationPhase.Backup_Finalize)
{
periodicOutput.SetFinished();
}
else if (phase == Duplicati.Library.Main.OperationPhase.Backup_PreBackupVerify)
{
output.MessageEvent("Checking remote backup ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Backup_PostBackupVerify)
{
output.MessageEvent("Checking remote backup ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Backup_PostBackupTest)
{
output.MessageEvent("Verifying remote backup ...");
}
else if (phase == Duplicati.Library.Main.OperationPhase.Backup_Compact)
{
output.MessageEvent("Compacting remote backup ...");
}
};
periodicOutput.WriteOutput += (progress, files, size, counting) => {
output.MessageEvent(string.Format(" {0} files need to be examined ({1}){2}", files, Library.Utility.Utility.FormatSizeString(size), counting ? " (still counting)" : ""));
};
using (var i = new Library.Main.Controller(backend, options, output))
result = i.Backup(dirs, filter);
}
if (output.VerboseOutput)
{
Library.Utility.Utility.PrintSerializeObject(result, Console.Out);
}
else
{
var parsedStats = result.BackendStatistics as Duplicati.Library.Interface.IParsedBackendStatistics;
output.MessageEvent(string.Format(" Duration of backup: {0:hh\\:mm\\:ss}", result.Duration));
if (parsedStats != null && parsedStats.KnownFileCount > 0)
{
output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount));
output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize)));
}
output.MessageEvent(string.Format(" Files added: {0}", result.AddedFiles));
output.MessageEvent(string.Format(" Files deleted: {0}", result.DeletedFiles));
output.MessageEvent(string.Format(" Files changed: {0}", result.ModifiedFiles));
output.MessageEvent(string.Format(" Data uploaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesUploaded)));
output.MessageEvent(string.Format(" Data downloaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesDownloaded)));
}
if (result.ExaminedFiles == 0 && (filter != null || !filter.Empty))
{
output.MessageEvent("No files were processed. If this was not intentional you may want to use the \"test-filters\" command");
}
output.MessageEvent("Backup completed successfully!");
//Interrupted = 50
if (result.PartialBackup)
{
return(50);
}
//Completed with errors = 3
if (result.ParsedResult == Library.Interface.ParsedResultType.Error)
{
return(3);
}
//Completed with warnings = 2
if (result.ParsedResult == Library.Interface.ParsedResultType.Warning)
{
return(2);
}
//Success, but no upload = 1
if (result.BackendStatistics.BytesUploaded == 0)
{
return(1);
}
return(0);
}
public void Run(string[] sources, Library.Utility.IFilter filter)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
using(m_database = new LocalBackupDatabase(m_options.Dbpath, m_options))
{
m_result.SetDatabase(m_database);
m_result.Dryrun = m_options.Dryrun;
Utility.VerifyParameters(m_database, m_options);
m_database.VerifyConsistency(null);
// If there is no filter, we set an empty filter to simplify the code
// If there is a filter, we make sure that the sources are included
m_filter = filter ?? new Library.Utility.FilterExpression();
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
var lastVolumeSize = -1L;
m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
System.Threading.Thread parallelScanner = null;
try
{
m_snapshot = GetSnapshot(sources, m_options, m_result);
// Start parallel scan
if (m_options.ChangedFilelist == null || m_options.ChangedFilelist.Length < 1)
{
parallelScanner = new System.Threading.Thread(CountFilesThread) {
Name = "Read ahead file counter",
IsBackground = true
};
parallelScanner.Start();
}
using(m_backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
using(m_filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
{
var incompleteFilesets = m_database.GetIncompleteFilesets(null).OrderBy(x => x.Value).ToArray();
if (incompleteFilesets.Length != 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreviousBackupFinalize);
m_result.AddMessage(string.Format("Uploading filelist from previous interrupted backup"));
using(var trn = m_database.BeginTransaction())
{
var incompleteSet = incompleteFilesets.Last();
var badIds = from n in incompleteFilesets select n.Key;
var prevs = (from n in m_database.FilesetTimes
where
n.Key < incompleteSet.Key
&&
!badIds.Contains(n.Key)
orderby n.Key
select n.Key).ToArray();
var prevId = prevs.Length == 0 ? -1 : prevs.Last();
FilesetVolumeWriter fsw = null;
try
{
var s = 1;
var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s);
var oldFilesetID = incompleteSet.Key;
// Probe for an unused filename
while (s < 60)
{
var id = m_database.GetRemoteVolumeID(VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, fileTime));
if (id < 0)
break;
fileTime = incompleteSet.Value + TimeSpan.FromSeconds(++s);
}
fsw = new FilesetVolumeWriter(m_options, fileTime);
fsw.VolumeID = m_database.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
var newFilesetID = m_database.CreateFileset(fsw.VolumeID, fileTime, trn);
m_database.LinkFilesetToVolume(newFilesetID, fsw.VolumeID, trn);
m_database.AppendFilesFromPreviousSet(trn, null, newFilesetID, prevId, fileTime);
m_database.WriteFileset(fsw, trn, newFilesetID);
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("Would upload fileset: {0}, size: {1}", fsw.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(fsw.LocalFilename).Length)));
}
else
{
m_database.UpdateRemoteVolume(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, trn);
using(new Logging.Timer("CommitUpdateFilelistVolume"))
trn.Commit();
m_backend.Put(fsw);
fsw = null;
}
}
finally
{
if (fsw != null)
try { fsw.Dispose(); }
catch { fsw = null; }
}
}
}
if (!m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupVerify);
using(new Logging.Timer("PreBackupVerify"))
{
try
{
FilelistProcessor.VerifyRemoteList(m_backend, m_options, m_database, m_result.BackendWriter);
}
catch (Exception ex)
{
if (m_options.AutoCleanup)
{
m_result.AddWarning("Backend verification failed, attempting automatic cleanup", ex);
m_result.RepairResults = new RepairResults(m_result);
new RepairHandler(m_backend.BackendUrl, m_options, (RepairResults)m_result.RepairResults).Run();
m_result.AddMessage("Backend cleanup finished, retrying verification");
FilelistProcessor.VerifyRemoteList(m_backend, m_options, m_database, m_result.BackendWriter);
}
else
throw;
}
}
}
m_database.BuildLookupTable(m_options);
m_transaction = m_database.BeginTransaction();
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
var filesetvolumeid = m_database.RegisterRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
m_database.CreateFileset(filesetvolumeid, VolumeBase.ParseFilename(m_filesetvolume.RemoteFilename).Time, m_transaction);
m_blockvolume = new BlockVolumeWriter(m_options);
m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction);
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
m_indexvolume = new IndexVolumeWriter(m_options);
m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction);
}
var filterhandler = new FilterHandler(m_snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy, m_result);
using(new Logging.Timer("BackupMainOperation"))
{
if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Length >= 1)
{
m_result.AddVerboseMessage("Processing supplied change list instead of enumerating filesystem");
m_result.OperationProgressUpdater.UpdatefileCount(m_options.ChangedFilelist.Length, 0, true);
foreach(var p in m_options.ChangedFilelist)
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
m_result.AddMessage("Stopping backup operation on request");
break;
}
FileAttributes fa = new FileAttributes();
try
{
fa = m_snapshot.GetAttributes(p);
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to read attributes: {0}, message: {1}", p, ex.Message), ex);
}
if (filterhandler.AttributeFilter(null, p, fa))
{
try
{
this.HandleFilesystemEntry(p, fa);
}
catch (Exception ex)
{
m_result.AddWarning(string.Format("Failed to process element: {0}, message: {1}", p, ex.Message), ex);
}
}
}
m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist);
}
else
{
foreach(var path in m_snapshot.EnumerateFilesAndFolders(filterhandler.AttributeFilter))
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
m_result.AddMessage("Stopping backup operation on request");
break;
}
this.HandleFilesystemEntry(path, m_snapshot.GetAttributes(path));
}
}
//If the scanner is still running for some reason, make sure we kill it now
if (parallelScanner != null && parallelScanner.IsAlive)
parallelScanner.Abort();
// We no longer need to snapshot active
try { m_snapshot.Dispose(); }
finally { m_snapshot = null; }
m_result.OperationProgressUpdater.UpdatefileCount(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles, true);
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Finalize);
using(new Logging.Timer("FinalizeRemoteVolumes"))
{
if (m_blockvolume.SourceSize > 0)
{
lastVolumeSize = m_blockvolume.SourceSize;
if (m_options.Dryrun)
{
m_result.AddDryrunMessage(string.Format("Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length)));
if (m_indexvolume != null)
{
m_blockvolume.Close();
UpdateIndexVolume();
m_indexvolume.FinishVolume(Library.Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
m_result.AddDryrunMessage(string.Format("Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)));
}
m_blockvolume.Dispose();
m_blockvolume = null;
m_indexvolume.Dispose();
m_indexvolume = null;
}
else
{
m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
m_blockvolume.Close();
UpdateIndexVolume();
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
m_backend.Put(m_blockvolume, m_indexvolume);
m_blockvolume = null;
m_indexvolume = null;
}
}
else
{
m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename, m_transaction);
if (m_indexvolume != null)
m_database.RemoveRemoteVolume(m_indexvolume.RemoteFilename, m_transaction);
}
}
using(new Logging.Timer("UpdateChangeStatistics"))
m_database.UpdateChangeStatistics(m_result);
using(new Logging.Timer("VerifyConsistency"))
m_database.VerifyConsistency(m_transaction);
var changeCount =
m_result.AddedFiles + m_result.ModifiedFiles + m_result.DeletedFiles +
m_result.AddedFolders + m_result.ModifiedFolders + m_result.DeletedFolders +
m_result.AddedSymlinks + m_result.ModifiedSymlinks + m_result.DeletedSymlinks;
//Changes in the filelist triggers a filelist upload
if (m_options.UploadUnchangedBackups || changeCount > 0)
{
using(new Logging.Timer("Uploading a new fileset"))
{
if (!string.IsNullOrEmpty(m_options.ControlFiles))
foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
m_filesetvolume.AddControlFile(p, m_options.GetCompressionHintFromFilename(p));
m_database.WriteFileset(m_filesetvolume, m_transaction);
m_filesetvolume.Close();
if (m_options.Dryrun)
m_result.AddDryrunMessage(string.Format("Would upload fileset volume: {0}, size: {1}", m_filesetvolume.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(m_filesetvolume.LocalFilename).Length)));
else
{
m_database.UpdateRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
using(new Logging.Timer("CommitUpdateRemoteVolume"))
m_transaction.Commit();
m_transaction = m_database.BeginTransaction();
m_backend.Put(m_filesetvolume);
}
}
}
else
{
m_result.AddVerboseMessage("removing temp files, as no data needs to be uploaded");
m_database.RemoveRemoteVolume(m_filesetvolume.RemoteFilename, m_transaction);
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
using(new Logging.Timer("Async backend wait"))
m_backend.WaitForComplete(m_database, m_transaction);
if (m_result.TaskControlRendevouz() != TaskControlState.Stop)
{
if (m_options.KeepTime.Ticks > 0 || m_options.KeepVersions != 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
m_result.DeleteResults = new DeleteResults(m_result);
using(var db = new LocalDeleteDatabase(m_database))
new DeleteHandler(m_backend.BackendUrl, m_options, (DeleteResults)m_result.DeleteResults).DoRun(db, m_transaction, true, lastVolumeSize <= m_options.SmallFileSize);
}
else if (lastVolumeSize <= m_options.SmallFileSize && !m_options.NoAutoCompact)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
m_result.CompactResults = new CompactResults(m_result);
using(var db = new LocalDeleteDatabase(m_database))
new CompactHandler(m_backend.BackendUrl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, m_transaction);
}
}
if (m_options.UploadVerificationFile)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
FilelistProcessor.UploadVerificationFile(m_backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
}
if (m_options.Dryrun)
{
m_transaction.Rollback();
m_transaction = null;
}
else
{
using(new Logging.Timer("CommitFinalizingBackup"))
m_transaction.Commit();
m_transaction = null;
m_database.Vacuum();
if (m_result.TaskControlRendevouz() != TaskControlState.Stop && !m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, m_database))
{
using(new Logging.Timer("AfterBackupVerify"))
FilelistProcessor.VerifyRemoteList(backend, m_options, m_database, m_result.BackendWriter);
backend.WaitForComplete(m_database, null);
}
if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
m_result.TestResults = new TestResults(m_result);
using(var testdb = new LocalTestDatabase(m_database))
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, testdb))
new TestHandler(m_backendurl, m_options, new TestResults(m_result))
.DoRun(m_options.BackupTestSampleCount, testdb, backend);
}
}
}
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
m_database.WriteResults();
return;
}
}
catch (Exception ex)
{
m_result.AddError("Fatal error", ex);
throw;
}
finally
{
if (parallelScanner != null && parallelScanner.IsAlive)
{
parallelScanner.Abort();
parallelScanner.Join(500);
if (parallelScanner.IsAlive)
m_result.AddWarning("Failed to terminate filecounter thread", null);
}
if (m_snapshot != null)
try { m_snapshot.Dispose(); }
catch (Exception ex) { m_result.AddError(string.Format("Failed to dispose snapshot"), ex); }
finally { m_snapshot = null; }
if (m_transaction != null)
try { m_transaction.Rollback(); }
catch (Exception ex) { m_result.AddError(string.Format("Rollback error: {0}", ex.Message), ex); }
}
}
}
public void Run(IEnumerable <string> filterstrings = null, Library.Utility.IFilter compositefilter = null)
{
var parsedfilter = new Library.Utility.FilterExpression(filterstrings);
var filter = Library.Utility.JoinedFilterExpression.Join(parsedfilter, compositefilter);
var simpleList = !((filter is FilterExpression expression && expression.Type == Library.Utility.FilterType.Simple) || m_options.AllVersions);
//Use a speedy local query
if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath))
{
using (var db = new Database.LocalListDatabase(m_options.Dbpath))
{
m_result.SetDatabase(db);
using (var filesets = db.SelectFileSets(m_options.Time, m_options.Version))
{
if (!filter.Empty)
{
if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions))
{
filesets.TakeFirst();
}
}
IEnumerable <Database.LocalListDatabase.IFileversion> files;
if (m_options.ListFolderContents)
{
files = filesets.SelectFolderContents(filter);
}
else if (m_options.ListPrefixOnly)
{
files = filesets.GetLargestPrefix(filter);
}
else if (filter.Empty)
{
files = null;
}
else
{
files = filesets.SelectFiles(filter);
}
if (m_options.ListSetsOnly)
{
m_result.SetResult(
filesets.QuickSets.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(),
null
);
}
else
{
m_result.SetResult(
filesets.Sets.Select(x =>
new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(),
files == null
? null
: (from n in files
select(Duplicati.Library.Interface.IListResultFile)(new ListResultFile(n.Path,
n.Sizes.ToArray())))
.ToArray()
);
}
return;
}
}
}
Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store");
//TODO: Add prefix and foldercontents
if (m_options.ListFolderContents)
{
throw new UserInformationException("Listing folder contents is not supported without a local database, consider using the \"repair\" option to rebuild the database.", "FolderContentListingRequiresLocalDatabase");
}
else if (m_options.ListPrefixOnly)
{
throw new UserInformationException("Listing prefixes is not supported without a local database, consider using the \"repair\" option to rebuild the database.", "PrefixListingRequiresLocalDatabase");
}
// Otherwise, grab info from remote location
using (var tmpdb = new Library.Utility.TempFile())
using (var db = new Database.LocalDatabase(tmpdb, "List", true))
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
{
m_result.SetDatabase(db);
var filteredList = ParseAndFilterFilesets(backend.List(), m_options);
if (filteredList.Count == 0)
{
throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder");
}
var numberSeq = CreateResultSequence(filteredList);
if (filter.Empty)
{
m_result.SetResult(numberSeq, null);
m_result.EncryptedFiles = filteredList.Any(x => !string.IsNullOrWhiteSpace(x.Value.EncryptionModule));
return;
}
var firstEntry = filteredList[0].Value;
filteredList.RemoveAt(0);
Dictionary <string, List <long> > res;
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
return;
}
using (var tmpfile = backend.Get(firstEntry.File.Name, firstEntry.File.Size, null))
using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(firstEntry.File.Name), tmpfile, m_options))
if (simpleList)
{
m_result.SetResult(
numberSeq.Take(1),
(from n in rd.Files
where Library.Utility.FilterExpression.Matches(filter, n.Path)
orderby n.Path
select new ListResultFile(n.Path, new long[] { n.Size }))
.ToArray()
);
return;
}
else
{
res = rd.Files
.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))
.ToDictionary(
x => x.Path,
y =>
{
var lst = new List <long>();
lst.Add(y.Size);
return(lst);
},
Library.Utility.Utility.ClientFilenameStringComparer
);
}
long flindex = 1;
foreach (var flentry in filteredList)
{
using (var tmpfile = backend.Get(flentry.Value.File.Name, flentry.Value.File == null ? -1 : flentry.Value.File.Size, null))
using (var rd = new Volumes.FilesetVolumeReader(flentry.Value.CompressionModule, tmpfile, m_options))
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
{
return;
}
foreach (var p in from n in rd.Files where Library.Utility.FilterExpression.Matches(filter, n.Path) select n)
{
List <long> lst;
if (!res.TryGetValue(p.Path, out lst))
{
lst = new List <long>();
res[p.Path] = lst;
for (var i = 0; i < flindex; i++)
{
lst.Add(-1);
}
}
lst.Add(p.Size);
}
foreach (var n in from i in res where i.Value.Count < flindex + 1 select i)
{
n.Value.Add(-1);
}
flindex++;
}
}
m_result.SetResult(
numberSeq,
from n in res
orderby n.Key
select(Duplicati.Library.Interface.IListResultFile)(new ListResultFile(n.Key, n.Value))
);
}
}
