public BackendManager(string backendurl, Options options, IBackendWriter statwriter, LocalDatabase database) { m_options = options; m_backendurl = backendurl; m_statwriter = statwriter; m_taskControl = statwriter as BasicResults; m_db = new DatabaseCollector(database, statwriter); m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); if (m_backend == null) throw new Exception(string.Format("Backend not supported: {0}", m_backendurl)); if (!m_options.NoEncryption) { m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); if (m_encryption == null) throw new Exception(string.Format("Encryption method not supported: ", m_options.EncryptionModule)); } if (m_taskControl != null) m_taskControl.StateChangedEvent += (state) => { if (state == TaskControlState.Abort) m_thread.Abort(); }; m_queue = new BlockingQueue<FileEntryItem>(options.SynchronousUpload ? 1 : (options.AsynchronousUploadLimit == 0 ? int.MaxValue : options.AsynchronousUploadLimit)); m_thread = new System.Threading.Thread(this.ThreadRun); m_thread.Name = "Backend Async Worker"; m_thread.IsBackground = true; m_thread.Start(); }
internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null) { string n = null; var opts = db.GetDbOptions(transaction); if(opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n))) options.RawOptions["blocksize"] = opts["blocksize"] + "b"; if (opts.ContainsKey("blockhash") && (!options.RawOptions.TryGetValue("block-hash-algorithm", out n) || string.IsNullOrEmpty(n))) options.RawOptions["block-hash-algorithm"] = opts["blockhash"]; if (opts.ContainsKey("filehash") && (!options.RawOptions.TryGetValue("file-hash-algorithm", out n) || string.IsNullOrEmpty(n))) options.RawOptions["file-hash-algorithm"] = opts["filehash"]; }
public Metahash(Dictionary<string, string> values, Options options) { m_values = values; var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm); if (hasher == null) throw new Exception(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm)); if (!hasher.CanReuseTransform) throw new Exception(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm)); using (var ms = new System.IO.MemoryStream()) using (var w = new StreamWriter(ms, Encoding.UTF8)) { w.Write(JsonConvert.SerializeObject(values)); w.Flush(); m_blob = ms.ToArray(); ms.Position = 0; m_hash = Convert.ToBase64String(hasher.ComputeHash(ms)); } }
public FilenameStrategy(Options options) : this(options.BackupPrefix, options.TimeSeparatorChar, options.UseShortFilenames, options.UseOldFilenames) { }
public static string GetDatabasePath(string backend, Options options, bool autoCreate = true, bool anyUsername = false) { if (options == null) options = new Options(new Dictionary<string, string>()); if (!string.IsNullOrEmpty(options.Dbpath)) return options.Dbpath; var folder = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Duplicati"); if (!System.IO.Directory.Exists(folder)) System.IO.Directory.CreateDirectory(folder); var file = System.IO.Path.Combine(folder, "dbconfig.json"); List<BackendEntry> configs; if (!System.IO.File.Exists(file)) configs = new List<BackendEntry>(); else configs = Newtonsoft.Json.JsonConvert.DeserializeObject<List<BackendEntry>>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8)); var uri = new Library.Utility.Uri(backend); string server = uri.Host; string path = uri.Path; string type = uri.Scheme; int port = uri.Port; string username = uri.Username; string password = uri.Password; string prefix = options.Prefix; if (username == null || password == null) { var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend); var ropts = new Dictionary<string, string>(options.RawOptions); foreach(var k in uri.QueryParameters.AllKeys) ropts[k] = uri.QueryParameters[k]; if (sopts != null) { foreach(var o in sopts) { if (username == null && o.Aliases != null && o.Aliases.Contains("auth-username", StringComparer.InvariantCultureIgnoreCase) && ropts.ContainsKey(o.Name)) username = ropts[o.Name]; if (password == null && o.Aliases != null && o.Aliases.Contains("auth-password", StringComparer.InvariantCultureIgnoreCase) && ropts.ContainsKey(o.Name)) password = ropts[o.Name]; } foreach(var o in sopts) { if (username == null && o.Name.Equals("auth-username", StringComparison.InvariantCultureIgnoreCase) && ropts.ContainsKey("auth-username")) username = ropts["auth-username"]; if (password == null && o.Name.Equals("auth-password", StringComparison.InvariantCultureIgnoreCase) && ropts.ContainsKey("auth-password")) password = ropts["auth-password"]; } } } if (password != null) password = Library.Utility.Utility.ByteArrayAsHexString(System.Security.Cryptography.SHA256.Create().ComputeHash(System.Text.Encoding.UTF8.GetBytes(password + "!" + uri.Scheme + "!" + uri.HostAndPath))); //Now find the one that matches :) var matches = (from n in configs where n.Type == type && //n.Passwordhash == password && n.Username == username && n.Port == port && n.Server == server && n.Path == path && n.Prefix == prefix select n).ToList(); if (matches.Count > 1) throw new Exception(string.Format("Multiple sources found for: {0}", backend)); // Re-select if (matches.Count == 0 && anyUsername && string.IsNullOrEmpty(username)) { matches = (from n in configs where n.Type == type && n.Port == port && n.Server == server && n.Path == path && n.Prefix == prefix select n).ToList(); if (matches.Count > 1) throw new Exception(String.Format("Multiple sources found for \"{0}\", try supplying --{1}", backend, "auth-username")); } if (matches.Count == 0 && !autoCreate) return null; if (matches.Count == 0) { var backupname = options.BackupName; if (string.IsNullOrEmpty(backupname) || backupname == Options.DefaultBackupName) backupname = GenerateRandomName(); else { foreach(var c in System.IO.Path.GetInvalidFileNameChars()) backupname = backupname.Replace(c.ToString(), ""); } var newpath = System.IO.Path.Combine(folder, backupname + ".sqlite"); int max_tries = 100; while (System.IO.File.Exists(newpath) && max_tries-- > 0) newpath = System.IO.Path.Combine(folder, GenerateRandomName()); if (System.IO.File.Exists(newpath)) throw new Exception("Unable to find a unique name for the database, please use --dbpath"); //Create a new one, add it to the list, and save it configs.Add(new BackendEntry() { Type = type, Server = server, Path = path, Prefix = prefix, Username = username, //Passwordhash = password, Port = port, Databasepath = newpath, ParameterFile = null }); var settings = new Newtonsoft.Json.JsonSerializerSettings(); settings.Formatting = Newtonsoft.Json.Formatting.Indented; System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8); return newpath; } else { return matches[0].Databasepath; } }
internal static void VerifyParameters(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null) { var newDict = new Dictionary<string, string>(); newDict.Add("blocksize", options.Blocksize.ToString()); newDict.Add("blockhash", options.BlockHashAlgorithm); newDict.Add("filehash", options.FileHashAlgorithm); var opts = db.GetDbOptions(transaction); if (options.NoEncryption) { newDict.Add("passphrase", "no-encryption"); } else { string salt; opts.TryGetValue("passphrase-salt", out salt); if (string.IsNullOrEmpty(salt)) { // Not Crypto-class PRNG salts var buf = new byte[32]; new Random().NextBytes(buf); //Add version so we can detect and change the algorithm salt = "v1:" + Library.Utility.Utility.ByteArrayAsHexString(buf); } newDict["passphrase-salt"] = salt; // We avoid storing the passphrase directly, // instead we salt and rehash repeatedly newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200))); } var needsUpdate = false; foreach(var k in newDict) if (!opts.ContainsKey(k.Key)) needsUpdate = true; else if (opts[k.Key] != k.Value) { if (k.Key == "passphrase") { if (!options.AllowPassphraseChange) { if (newDict[k.Key] == "no-encryption") throw new Exception("Unsupported removal of passphrase"); else if (opts[k.Key] == "no-encryption") throw new Exception("Unsupported addition of passphrase"); else throw new Exception("Unsupported change of passphrase"); } } else throw new Exception(string.Format("Unsupported change of parameter \"{0}\" from \"{1}\" to \"{2}\"", k.Key, opts[k.Key], k.Value)); } //Extra sanity check if (db.GetBlocksLargerThan(options.Blocksize) > 0) throw new Exception("Unsupported block-size change detected"); if (needsUpdate) db.SetDbOptions(newDict, transaction); }
/// <summary> /// Constructs a container for a given metadata dictionary /// </summary> /// <param name="values">The metadata values to wrap</param> /// <returns>A IMetahash instance</returns> public static IMetahash WrapMetadata(Dictionary<string, string> values, Options options) { return new Metahash(values, options); }
public bool UpdateHashAndSize(Options options) { if (Hash == null || Size < 0) { Hash = CalculateFileHash(this.LocalFilename); Size = new System.IO.FileInfo(this.LocalFilename).Length; return true; } return false; }
/// <summary> /// Constructs a new BackendWrapper /// </summary> /// <param name="statistics">The statistics logging module, may be null</param> /// <param name="backend">The url to the backend to wrap</param> /// <param name="options">A set of backend options</param> public BackendWrapper(CommunicationStatistics statistics, string backend, Options options) { m_statistics = statistics; m_options = options; m_filenamestrategy = new FilenameStrategy(m_options); m_backendUrl = backend; m_backend = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(backend, m_options.RawOptions); if (m_backend == null) throw new Exception(string.Format(Strings.BackendWrapper.BackendNotFoundError, backend)); m_reuse_backend = !m_options.NoConnectionReuse; m_first_backend_use = true; m_backendSupportsCreateFolder = m_backend is Library.Interface.IBackend_v2; if (m_options.AutoCleanup) m_orphans = new List<BackupEntryBase>(); if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && !System.IO.Directory.Exists(m_options.SignatureCachePath)) System.IO.Directory.CreateDirectory(m_options.SignatureCachePath); if (!string.IsNullOrEmpty(m_options.Backendlogdatabase)) { m_backendInterfaceLogger = new StateVerification.StateDatabase(m_options.Backendlogdatabase, statistics); m_backendInterfaceLogger.BeginOperation(m_options.MainAction.ToString()); } m_async = m_options.AsynchronousUpload; if (m_async) { //If we are using async operations, the entire class is actually threadsafe, //utilizing a common exclusive lock on all operations. But the implementation does //not prevent starvation, so it should not be called by multiple threads. m_pendingOperations = new Queue<KeyValuePair<BackupEntryBase, string>>(); m_asyncItemProcessed = new System.Threading.AutoResetEvent(false); m_asyncItemReady = new System.Threading.AutoResetEvent(false); m_workerThread = new System.Threading.Thread(ProcessQueue); m_workerThread.Name = "AsyncUploaderThread"; m_queuelock = new object(); m_workerThread.Start(); } }
public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter) { if (args.Count != 4) { Console.WriteLine("Invalid argument count ({0} expected 4): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args)); return 100; } string target_compr_module = args[1]; if (!Library.DynamicLoader.CompressionLoader.Keys.Contains(target_compr_module)) { Console.WriteLine("Target compression module not found: {0}{1}Modules supported: {2}", args[1], Environment.NewLine, string.Join(", ", Library.DynamicLoader.CompressionLoader.Keys)); return 100; } var m_Options = new Options(options); using (var backend = Library.DynamicLoader.BackendLoader.GetBackend(args[2], options)) { if (backend == null) { Console.WriteLine("Backend not found: {0}{1}Backends supported: {2}", args[2], Environment.NewLine, string.Join(", ", Library.DynamicLoader.BackendLoader.Keys)); return 100; } var targetfolder = Path.GetFullPath(args[3]); if (!Directory.Exists(args[3])) { Console.WriteLine("Creating target folder: {0}", targetfolder); Directory.CreateDirectory(targetfolder); } Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey); var rawlist = backend.List(); Console.WriteLine("Found {0} files", rawlist.Count); var i = 0; var downloaded = 0; var errors = 0; var needspass = 0; var remotefiles = (from x in rawlist let n = VolumeBase.ParseFilename(x) where n != null && n.Prefix == m_Options.Prefix select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times if (remotefiles.Length == 0) { if (rawlist.Count == 0) Console.WriteLine("No files were found at the remote location, perhaps the target url is incorrect?"); else { var tmp = (from x in rawlist let n = VolumeBase.ParseFilename(x) where n != null select n.Prefix).ToArray(); var types = tmp.Distinct().ToArray(); if (tmp.Length == 0) Console.WriteLine("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count); else if (types.Length == 1) Console.WriteLine("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0]); else Console.WriteLine("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types)); } return 100; } bool reencrypt = Library.Utility.Utility.ParseBoolOption(options, "reencrypt"); bool reupload = Library.Utility.Utility.ParseBoolOption(options, "reupload"); foreach (var entry in remotefiles) { try { Console.Write("{0}: {1}", i, entry.File.Name); var local = Path.Combine(targetfolder, entry.File.Name); if (entry.EncryptionModule != null) { if (string.IsNullOrWhiteSpace(m_Options.Passphrase)) { needspass++; Console.WriteLine(" - No passphrase supplied, skipping"); continue; } local = local.Substring(0, local.Length - entry.EncryptionModule.Length - 1); } if (entry.CompressionModule == target_compr_module) { Console.WriteLine(" - compression types are same"); continue; } string localNew; if (entry.CompressionModule != null) { localNew = local.Substring(0, local.Length - entry.CompressionModule.Length - 1) + "." + target_compr_module; if (File.Exists(localNew)) { Console.WriteLine(" - target file already exist"); continue; } } else { Console.WriteLine(" - cannot detect compression type"); continue; } if (File.Exists(local)) File.Delete(local); Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(entry.File.Size)); DateTime originLastWriteTime; FileInfo destinationFileInfo; using (var tf = new Library.Utility.TempFile()) { backend.Get(entry.File.Name, tf); originLastWriteTime = new FileInfo(tf).LastWriteTime; downloaded++; if (entry.EncryptionModule != null) { Console.Write(" decrypting ..."); using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options)) using (var tf2 = new Library.Utility.TempFile()) { m.Decrypt(tf, tf2); File.Copy(tf2, local); File.Delete(tf2); } } else File.Copy(tf, local); File.Delete(tf); destinationFileInfo = new FileInfo(local); destinationFileInfo.LastWriteTime = originLastWriteTime; } if (entry.CompressionModule != null) { Console.Write(" recompressing ..."); using (var cmOld = Library.DynamicLoader.CompressionLoader.GetModule(entry.CompressionModule, local, options)) using (var cmNew = Library.DynamicLoader.CompressionLoader.GetModule(target_compr_module, localNew, options)) foreach (var cmfile in cmOld.ListFiles("")) { string cmfileNew = cmfile; if (entry.FileType == RemoteVolumeType.Index) { var cmFileVolume = Library.Main.Volumes.VolumeBase.ParseFilename(cmfileNew); if (cmFileVolume != null) { cmfileNew = cmfileNew.Replace("." + cmFileVolume.CompressionModule, "." + target_compr_module); if(!reencrypt) cmfileNew = cmfileNew.Replace("." + cmFileVolume.EncryptionModule, ""); } } using (var sourceStream = cmOld.OpenRead(cmfile)) using (var cs = cmNew.CreateFile(cmfileNew, Duplicati.Library.Interface.CompressionHint.Compressible, cmOld.GetLastWriteTime(cmfile))) Library.Utility.Utility.CopyStream(sourceStream, cs); } File.Delete(local); destinationFileInfo = new FileInfo(localNew); destinationFileInfo.LastWriteTime = originLastWriteTime; } if (reencrypt && entry.EncryptionModule != null) { Console.Write(" reencrypting ..."); using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options)) { m.Encrypt(localNew, localNew + "." + m.FilenameExtension); File.Delete(localNew); localNew = localNew + "." + m.FilenameExtension; } destinationFileInfo = new FileInfo(localNew); destinationFileInfo.LastWriteTime = originLastWriteTime; } if (reupload) { backend.Put((new FileInfo(localNew)).Name, localNew); backend.Delete(entry.File.Name); File.Delete(localNew); } Console.WriteLine(" done!"); } catch (Exception ex) { Console.WriteLine(" error: {0}", ex.ToString()); errors++; } i++; } if (reupload) { var remoteverificationfileexist = rawlist.Any(x => x.Name == (m_Options.Prefix + "-verification.json")); if (remoteverificationfileexist) { Console.WriteLine("Found verification file {0} - deleting", m_Options.Prefix + "-verification.json"); backend.Delete(m_Options.Prefix + "-verification.json"); } } if (needspass > 0 && downloaded == 0) { Console.WriteLine("No files downloaded, try adding --passphrase to decrypt files"); return 100; } Console.WriteLine("Download complete, of {0} remote files, {1} were downloaded with {2} errors", remotefiles.Count(), downloaded, errors); if (needspass > 0) Console.WriteLine("Additonally {0} remote files were skipped because of encryption, supply --passphrase to download those"); if (errors > 0) { Console.WriteLine("There were errors during recompress of remote backend files!"); return 200; } return 0; } }
public static void PurgeSignatureCache(Dictionary<string, string> options) { Options opt = new Options(options); if (string.IsNullOrEmpty(opt.SignatureCachePath)) throw new Exception(Strings.Interface.SignatureCachePathMissingError); else System.IO.Directory.Delete(opt.SignatureCachePath, true); }
/// <summary> /// Constructs a new interface for performing backup and restore operations /// </summary> /// <param name="backend">The url for the backend to use</param> /// <param name="options">All required options</param> public Interface(string backend, Dictionary<string, string> options) { m_backend = backend; m_options = new Options(options); m_liveControl = new LiveControl.LiveControl(System.Threading.Thread.CurrentThread, m_options); OperationProgress += new OperationProgressEvent(Interface_OperationProgress); }