public BackendManager(string backendurl, Options options, IBackendWriter statwriter, LocalDatabase database)
{
m_options = options;
m_backendurl = backendurl;
m_statwriter = statwriter;
m_taskControl = statwriter as BasicResults;
m_db = new DatabaseCollector(database, statwriter);
m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions);
if (m_backend == null)
throw new Exception(string.Format("Backend not supported: {0}", m_backendurl));
if (!m_options.NoEncryption)
{
m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions);
if (m_encryption == null)
throw new Exception(string.Format("Encryption method not supported: ", m_options.EncryptionModule));
}
if (m_taskControl != null)
m_taskControl.StateChangedEvent += (state) => {
if (state == TaskControlState.Abort)
m_thread.Abort();
};
m_queue = new BlockingQueue<FileEntryItem>(options.SynchronousUpload ? 1 : (options.AsynchronousUploadLimit == 0 ? int.MaxValue : options.AsynchronousUploadLimit));
m_thread = new System.Threading.Thread(this.ThreadRun);
m_thread.Name = "Backend Async Worker";
m_thread.IsBackground = true;
m_thread.Start();
}
internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
{
string n = null;
var opts = db.GetDbOptions(transaction);
if(opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n)))
options.RawOptions["blocksize"] = opts["blocksize"] + "b";
if (opts.ContainsKey("blockhash") && (!options.RawOptions.TryGetValue("block-hash-algorithm", out n) || string.IsNullOrEmpty(n)))
options.RawOptions["block-hash-algorithm"] = opts["blockhash"];
if (opts.ContainsKey("filehash") && (!options.RawOptions.TryGetValue("file-hash-algorithm", out n) || string.IsNullOrEmpty(n)))
options.RawOptions["file-hash-algorithm"] = opts["filehash"];
}
public Metahash(Dictionary<string, string> values, Options options)
{
m_values = values;
var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm);
if (hasher == null)
throw new Exception(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
if (!hasher.CanReuseTransform)
throw new Exception(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm));
using (var ms = new System.IO.MemoryStream())
using (var w = new StreamWriter(ms, Encoding.UTF8))
{
w.Write(JsonConvert.SerializeObject(values));
w.Flush();
m_blob = ms.ToArray();
ms.Position = 0;
m_hash = Convert.ToBase64String(hasher.ComputeHash(ms));
}
}
public FilenameStrategy(Options options)
: this(options.BackupPrefix, options.TimeSeparatorChar, options.UseShortFilenames, options.UseOldFilenames)
{
}
public static string GetDatabasePath(string backend, Options options, bool autoCreate = true, bool anyUsername = false)
{
if (options == null)
options = new Options(new Dictionary<string, string>());
if (!string.IsNullOrEmpty(options.Dbpath))
return options.Dbpath;
var folder = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Duplicati");
if (!System.IO.Directory.Exists(folder))
System.IO.Directory.CreateDirectory(folder);
var file = System.IO.Path.Combine(folder, "dbconfig.json");
List<BackendEntry> configs;
if (!System.IO.File.Exists(file))
configs = new List<BackendEntry>();
else
configs = Newtonsoft.Json.JsonConvert.DeserializeObject<List<BackendEntry>>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8));
var uri = new Library.Utility.Uri(backend);
string server = uri.Host;
string path = uri.Path;
string type = uri.Scheme;
int port = uri.Port;
string username = uri.Username;
string password = uri.Password;
string prefix = options.Prefix;
if (username == null || password == null)
{
var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend);
var ropts = new Dictionary<string, string>(options.RawOptions);
foreach(var k in uri.QueryParameters.AllKeys)
ropts[k] = uri.QueryParameters[k];
if (sopts != null)
{
foreach(var o in sopts)
{
if (username == null && o.Aliases != null && o.Aliases.Contains("auth-username", StringComparer.InvariantCultureIgnoreCase) && ropts.ContainsKey(o.Name))
username = ropts[o.Name];
if (password == null && o.Aliases != null && o.Aliases.Contains("auth-password", StringComparer.InvariantCultureIgnoreCase) && ropts.ContainsKey(o.Name))
password = ropts[o.Name];
}
foreach(var o in sopts)
{
if (username == null && o.Name.Equals("auth-username", StringComparison.InvariantCultureIgnoreCase) && ropts.ContainsKey("auth-username"))
username = ropts["auth-username"];
if (password == null && o.Name.Equals("auth-password", StringComparison.InvariantCultureIgnoreCase) && ropts.ContainsKey("auth-password"))
password = ropts["auth-password"];
}
}
}
if (password != null)
password = Library.Utility.Utility.ByteArrayAsHexString(System.Security.Cryptography.SHA256.Create().ComputeHash(System.Text.Encoding.UTF8.GetBytes(password + "!" + uri.Scheme + "!" + uri.HostAndPath)));
//Now find the one that matches :)
var matches = (from n in configs
where
n.Type == type &&
//n.Passwordhash == password &&
n.Username == username &&
n.Port == port &&
n.Server == server &&
n.Path == path &&
n.Prefix == prefix
select n).ToList();
if (matches.Count > 1)
throw new Exception(string.Format("Multiple sources found for: {0}", backend));
// Re-select
if (matches.Count == 0 && anyUsername && string.IsNullOrEmpty(username))
{
matches = (from n in configs
where
n.Type == type &&
n.Port == port &&
n.Server == server &&
n.Path == path &&
n.Prefix == prefix
select n).ToList();
if (matches.Count > 1)
throw new Exception(String.Format("Multiple sources found for \"{0}\", try supplying --{1}", backend, "auth-username"));
}
if (matches.Count == 0 && !autoCreate)
return null;
if (matches.Count == 0)
{
var backupname = options.BackupName;
if (string.IsNullOrEmpty(backupname) || backupname == Options.DefaultBackupName)
backupname = GenerateRandomName();
else
{
foreach(var c in System.IO.Path.GetInvalidFileNameChars())
backupname = backupname.Replace(c.ToString(), "");
}
var newpath = System.IO.Path.Combine(folder, backupname + ".sqlite");
int max_tries = 100;
while (System.IO.File.Exists(newpath) && max_tries-- > 0)
newpath = System.IO.Path.Combine(folder, GenerateRandomName());
if (System.IO.File.Exists(newpath))
throw new Exception("Unable to find a unique name for the database, please use --dbpath");
//Create a new one, add it to the list, and save it
configs.Add(new BackendEntry() {
Type = type,
Server = server,
Path = path,
Prefix = prefix,
Username = username,
//Passwordhash = password,
Port = port,
Databasepath = newpath,
ParameterFile = null
});
var settings = new Newtonsoft.Json.JsonSerializerSettings();
settings.Formatting = Newtonsoft.Json.Formatting.Indented;
System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8);
return newpath;
}
else
{
return matches[0].Databasepath;
}
}
internal static void VerifyParameters(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
{
var newDict = new Dictionary<string, string>();
newDict.Add("blocksize", options.Blocksize.ToString());
newDict.Add("blockhash", options.BlockHashAlgorithm);
newDict.Add("filehash", options.FileHashAlgorithm);
var opts = db.GetDbOptions(transaction);
if (options.NoEncryption)
{
newDict.Add("passphrase", "no-encryption");
}
else
{
string salt;
opts.TryGetValue("passphrase-salt", out salt);
if (string.IsNullOrEmpty(salt))
{
// Not Crypto-class PRNG salts
var buf = new byte[32];
new Random().NextBytes(buf);
//Add version so we can detect and change the algorithm
salt = "v1:" + Library.Utility.Utility.ByteArrayAsHexString(buf);
}
newDict["passphrase-salt"] = salt;
// We avoid storing the passphrase directly,
// instead we salt and rehash repeatedly
newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200)));
}
var needsUpdate = false;
foreach(var k in newDict)
if (!opts.ContainsKey(k.Key))
needsUpdate = true;
else if (opts[k.Key] != k.Value)
{
if (k.Key == "passphrase")
{
if (!options.AllowPassphraseChange)
{
if (newDict[k.Key] == "no-encryption")
throw new Exception("Unsupported removal of passphrase");
else if (opts[k.Key] == "no-encryption")
throw new Exception("Unsupported addition of passphrase");
else
throw new Exception("Unsupported change of passphrase");
}
}
else
throw new Exception(string.Format("Unsupported change of parameter \"{0}\" from \"{1}\" to \"{2}\"", k.Key, opts[k.Key], k.Value));
}
//Extra sanity check
if (db.GetBlocksLargerThan(options.Blocksize) > 0)
throw new Exception("Unsupported block-size change detected");
if (needsUpdate)
db.SetDbOptions(newDict, transaction);
}
/// <summary>
/// Constructs a container for a given metadata dictionary
/// </summary>
/// <param name="values">The metadata values to wrap</param>
/// <returns>A IMetahash instance</returns>
public static IMetahash WrapMetadata(Dictionary<string, string> values, Options options)
{
return new Metahash(values, options);
}
public bool UpdateHashAndSize(Options options)
{
if (Hash == null || Size < 0)
{
Hash = CalculateFileHash(this.LocalFilename);
Size = new System.IO.FileInfo(this.LocalFilename).Length;
return true;
}
return false;
}
/// <summary>
/// Constructs a new BackendWrapper
/// </summary>
/// <param name="statistics">The statistics logging module, may be null</param>
/// <param name="backend">The url to the backend to wrap</param>
/// <param name="options">A set of backend options</param>
public BackendWrapper(CommunicationStatistics statistics, string backend, Options options)
{
m_statistics = statistics;
m_options = options;
m_filenamestrategy = new FilenameStrategy(m_options);
m_backendUrl = backend;
m_backend = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(backend, m_options.RawOptions);
if (m_backend == null)
throw new Exception(string.Format(Strings.BackendWrapper.BackendNotFoundError, backend));
m_reuse_backend = !m_options.NoConnectionReuse;
m_first_backend_use = true;
m_backendSupportsCreateFolder = m_backend is Library.Interface.IBackend_v2;
if (m_options.AutoCleanup)
m_orphans = new List<BackupEntryBase>();
if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && !System.IO.Directory.Exists(m_options.SignatureCachePath))
System.IO.Directory.CreateDirectory(m_options.SignatureCachePath);
if (!string.IsNullOrEmpty(m_options.Backendlogdatabase))
{
m_backendInterfaceLogger = new StateVerification.StateDatabase(m_options.Backendlogdatabase, statistics);
m_backendInterfaceLogger.BeginOperation(m_options.MainAction.ToString());
}
m_async = m_options.AsynchronousUpload;
if (m_async)
{
//If we are using async operations, the entire class is actually threadsafe,
//utilizing a common exclusive lock on all operations. But the implementation does
//not prevent starvation, so it should not be called by multiple threads.
m_pendingOperations = new Queue<KeyValuePair<BackupEntryBase, string>>();
m_asyncItemProcessed = new System.Threading.AutoResetEvent(false);
m_asyncItemReady = new System.Threading.AutoResetEvent(false);
m_workerThread = new System.Threading.Thread(ProcessQueue);
m_workerThread.Name = "AsyncUploaderThread";
m_queuelock = new object();
m_workerThread.Start();
}
}
public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
{
if (args.Count != 4)
{
Console.WriteLine("Invalid argument count ({0} expected 4): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
return 100;
}
string target_compr_module = args[1];
if (!Library.DynamicLoader.CompressionLoader.Keys.Contains(target_compr_module))
{
Console.WriteLine("Target compression module not found: {0}{1}Modules supported: {2}", args[1], Environment.NewLine, string.Join(", ", Library.DynamicLoader.CompressionLoader.Keys));
return 100;
}
var m_Options = new Options(options);
using (var backend = Library.DynamicLoader.BackendLoader.GetBackend(args[2], options))
{
if (backend == null)
{
Console.WriteLine("Backend not found: {0}{1}Backends supported: {2}", args[2], Environment.NewLine, string.Join(", ", Library.DynamicLoader.BackendLoader.Keys));
return 100;
}
var targetfolder = Path.GetFullPath(args[3]);
if (!Directory.Exists(args[3]))
{
Console.WriteLine("Creating target folder: {0}", targetfolder);
Directory.CreateDirectory(targetfolder);
}
Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey);
var rawlist = backend.List();
Console.WriteLine("Found {0} files", rawlist.Count);
var i = 0;
var downloaded = 0;
var errors = 0;
var needspass = 0;
var remotefiles =
(from x in rawlist
let n = VolumeBase.ParseFilename(x)
where n != null && n.Prefix == m_Options.Prefix
select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times
if (remotefiles.Length == 0)
{
if (rawlist.Count == 0)
Console.WriteLine("No files were found at the remote location, perhaps the target url is incorrect?");
else
{
var tmp =
(from x in rawlist
let n = VolumeBase.ParseFilename(x)
where
n != null
select n.Prefix).ToArray();
var types = tmp.Distinct().ToArray();
if (tmp.Length == 0)
Console.WriteLine("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count);
else if (types.Length == 1)
Console.WriteLine("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0]);
else
Console.WriteLine("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types));
}
return 100;
}
bool reencrypt = Library.Utility.Utility.ParseBoolOption(options, "reencrypt");
bool reupload = Library.Utility.Utility.ParseBoolOption(options, "reupload");
foreach (var entry in remotefiles)
{
try
{
Console.Write("{0}: {1}", i, entry.File.Name);
var local = Path.Combine(targetfolder, entry.File.Name);
if (entry.EncryptionModule != null)
{
if (string.IsNullOrWhiteSpace(m_Options.Passphrase))
{
needspass++;
Console.WriteLine(" - No passphrase supplied, skipping");
continue;
}
local = local.Substring(0, local.Length - entry.EncryptionModule.Length - 1);
}
if (entry.CompressionModule == target_compr_module)
{
Console.WriteLine(" - compression types are same");
continue;
}
string localNew;
if (entry.CompressionModule != null)
{
localNew = local.Substring(0, local.Length - entry.CompressionModule.Length - 1) + "." + target_compr_module;
if (File.Exists(localNew))
{
Console.WriteLine(" - target file already exist");
continue;
}
}
else
{
Console.WriteLine(" - cannot detect compression type");
continue;
}
if (File.Exists(local))
File.Delete(local);
Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(entry.File.Size));
DateTime originLastWriteTime;
FileInfo destinationFileInfo;
using (var tf = new Library.Utility.TempFile())
{
backend.Get(entry.File.Name, tf);
originLastWriteTime = new FileInfo(tf).LastWriteTime;
downloaded++;
if (entry.EncryptionModule != null)
{
Console.Write(" decrypting ...");
using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options))
using (var tf2 = new Library.Utility.TempFile())
{
m.Decrypt(tf, tf2);
File.Copy(tf2, local);
File.Delete(tf2);
}
}
else
File.Copy(tf, local);
File.Delete(tf);
destinationFileInfo = new FileInfo(local);
destinationFileInfo.LastWriteTime = originLastWriteTime;
}
if (entry.CompressionModule != null)
{
Console.Write(" recompressing ...");
using (var cmOld = Library.DynamicLoader.CompressionLoader.GetModule(entry.CompressionModule, local, options))
using (var cmNew = Library.DynamicLoader.CompressionLoader.GetModule(target_compr_module, localNew, options))
foreach (var cmfile in cmOld.ListFiles(""))
{
string cmfileNew = cmfile;
if (entry.FileType == RemoteVolumeType.Index)
{
var cmFileVolume = Library.Main.Volumes.VolumeBase.ParseFilename(cmfileNew);
if (cmFileVolume != null)
{
cmfileNew = cmfileNew.Replace("." + cmFileVolume.CompressionModule, "." + target_compr_module);
if(!reencrypt)
cmfileNew = cmfileNew.Replace("." + cmFileVolume.EncryptionModule, "");
}
}
using (var sourceStream = cmOld.OpenRead(cmfile))
using (var cs = cmNew.CreateFile(cmfileNew, Duplicati.Library.Interface.CompressionHint.Compressible, cmOld.GetLastWriteTime(cmfile)))
Library.Utility.Utility.CopyStream(sourceStream, cs);
}
File.Delete(local);
destinationFileInfo = new FileInfo(localNew);
destinationFileInfo.LastWriteTime = originLastWriteTime;
}
if (reencrypt && entry.EncryptionModule != null)
{
Console.Write(" reencrypting ...");
using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options))
{
m.Encrypt(localNew, localNew + "." + m.FilenameExtension);
File.Delete(localNew);
localNew = localNew + "." + m.FilenameExtension;
}
destinationFileInfo = new FileInfo(localNew);
destinationFileInfo.LastWriteTime = originLastWriteTime;
}
if (reupload)
{
backend.Put((new FileInfo(localNew)).Name, localNew);
backend.Delete(entry.File.Name);
File.Delete(localNew);
}
Console.WriteLine(" done!");
}
catch (Exception ex)
{
Console.WriteLine(" error: {0}", ex.ToString());
errors++;
}
i++;
}
if (reupload)
{
var remoteverificationfileexist = rawlist.Any(x => x.Name == (m_Options.Prefix + "-verification.json"));
if (remoteverificationfileexist)
{
Console.WriteLine("Found verification file {0} - deleting", m_Options.Prefix + "-verification.json");
backend.Delete(m_Options.Prefix + "-verification.json");
}
}
if (needspass > 0 && downloaded == 0)
{
Console.WriteLine("No files downloaded, try adding --passphrase to decrypt files");
return 100;
}
Console.WriteLine("Download complete, of {0} remote files, {1} were downloaded with {2} errors", remotefiles.Count(), downloaded, errors);
if (needspass > 0)
Console.WriteLine("Additonally {0} remote files were skipped because of encryption, supply --passphrase to download those");
if (errors > 0)
{
Console.WriteLine("There were errors during recompress of remote backend files!");
return 200;
}
return 0;
}
}
public static void PurgeSignatureCache(Dictionary<string, string> options)
{
Options opt = new Options(options);
if (string.IsNullOrEmpty(opt.SignatureCachePath))
throw new Exception(Strings.Interface.SignatureCachePathMissingError);
else
System.IO.Directory.Delete(opt.SignatureCachePath, true);
}
/// <summary>
/// Constructs a new interface for performing backup and restore operations
/// </summary>
/// <param name="backend">The url for the backend to use</param>
/// <param name="options">All required options</param>
public Interface(string backend, Dictionary<string, string> options)
{
m_backend = backend;
m_options = new Options(options);
m_liveControl = new LiveControl.LiveControl(System.Threading.Thread.CurrentThread, m_options);
OperationProgress += new OperationProgressEvent(Interface_OperationProgress);
}