Beispiel #1
0
        public BackendManager(string backendurl, Options options, IBackendWriter statwriter, LocalDatabase database)
        {
            m_options = options;
            m_backendurl = backendurl;
            m_statwriter = statwriter;
            m_taskControl = statwriter as BasicResults;
            m_db = new DatabaseCollector(database, statwriter);

            m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions);
            if (m_backend == null)
                throw new Exception(string.Format("Backend not supported: {0}", m_backendurl));

            if (!m_options.NoEncryption)
            {
                m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions);
                if (m_encryption == null)
                    throw new Exception(string.Format("Encryption method not supported: ", m_options.EncryptionModule));
            }

            if (m_taskControl != null)
                m_taskControl.StateChangedEvent += (state) => {
                    if (state == TaskControlState.Abort)
                        m_thread.Abort();
                };
            m_queue = new BlockingQueue<FileEntryItem>(options.SynchronousUpload ? 1 : (options.AsynchronousUploadLimit == 0 ? int.MaxValue : options.AsynchronousUploadLimit));
            m_thread = new System.Threading.Thread(this.ThreadRun);
            m_thread.Name = "Backend Async Worker";
            m_thread.IsBackground = true;
            m_thread.Start();
        }
Beispiel #2
0
        internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
        {
            string n = null;
            var opts = db.GetDbOptions(transaction);
            if(opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n)))
                options.RawOptions["blocksize"] = opts["blocksize"] + "b";

            if (opts.ContainsKey("blockhash") && (!options.RawOptions.TryGetValue("block-hash-algorithm", out n) || string.IsNullOrEmpty(n)))
                options.RawOptions["block-hash-algorithm"] = opts["blockhash"];
            if (opts.ContainsKey("filehash") && (!options.RawOptions.TryGetValue("file-hash-algorithm", out n) || string.IsNullOrEmpty(n)))
                options.RawOptions["file-hash-algorithm"] = opts["filehash"];
        }
Beispiel #3
0
         public Metahash(Dictionary<string, string> values, Options options)
         {
             m_values = values;
             var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.BlockHashAlgorithm);
             if (hasher == null)
                 throw new Exception(Strings.Common.InvalidHashAlgorithm(options.BlockHashAlgorithm));
             if (!hasher.CanReuseTransform)
                 throw new Exception(Strings.Common.InvalidCryptoSystem(options.BlockHashAlgorithm));
                 
             using (var ms = new System.IO.MemoryStream())
             using (var w = new StreamWriter(ms, Encoding.UTF8))
             {
                 w.Write(JsonConvert.SerializeObject(values));
                 w.Flush();
 
                 m_blob = ms.ToArray();
 
                 ms.Position = 0;
                 m_hash = Convert.ToBase64String(hasher.ComputeHash(ms));
             }
         }
 public FilenameStrategy(Options options)
     : this(options.BackupPrefix, options.TimeSeparatorChar, options.UseShortFilenames, options.UseOldFilenames)
 {
 }
Beispiel #5
0
        public static string GetDatabasePath(string backend, Options options, bool autoCreate = true, bool anyUsername = false)
        {
            if (options == null)
                options = new Options(new Dictionary<string, string>());

            if (!string.IsNullOrEmpty(options.Dbpath))
                return options.Dbpath;
         
            var folder = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Duplicati");
            if (!System.IO.Directory.Exists(folder))
                System.IO.Directory.CreateDirectory(folder);
                
            var file = System.IO.Path.Combine(folder, "dbconfig.json");
            List<BackendEntry> configs;
            if (!System.IO.File.Exists(file))
                configs = new List<BackendEntry>();
            else
                configs = Newtonsoft.Json.JsonConvert.DeserializeObject<List<BackendEntry>>(System.IO.File.ReadAllText(file, System.Text.Encoding.UTF8));
            
            var uri = new Library.Utility.Uri(backend);
            string server = uri.Host;
            string path = uri.Path;
            string type = uri.Scheme;
            int port = uri.Port;
            string username = uri.Username;
            string password = uri.Password;
            string prefix = options.Prefix;
            
            if (username == null || password == null)
            {
                var sopts = DynamicLoader.BackendLoader.GetSupportedCommands(backend);
                var ropts = new Dictionary<string, string>(options.RawOptions);
                foreach(var k in uri.QueryParameters.AllKeys)
                    ropts[k] = uri.QueryParameters[k];
                
                if (sopts != null)
                {
                    foreach(var o in sopts)
                    {
                        if (username == null && o.Aliases != null && o.Aliases.Contains("auth-username", StringComparer.InvariantCultureIgnoreCase) && ropts.ContainsKey(o.Name))
                            username = ropts[o.Name];
                        if (password == null && o.Aliases != null && o.Aliases.Contains("auth-password", StringComparer.InvariantCultureIgnoreCase) && ropts.ContainsKey(o.Name))
                            password = ropts[o.Name];
                    }
                
                    foreach(var o in sopts)
                    {
                        if (username == null && o.Name.Equals("auth-username", StringComparison.InvariantCultureIgnoreCase) && ropts.ContainsKey("auth-username"))
                            username = ropts["auth-username"];
                        if (password == null && o.Name.Equals("auth-password", StringComparison.InvariantCultureIgnoreCase) && ropts.ContainsKey("auth-password"))
                            password = ropts["auth-password"];
                    }
                }
            }
            
            if (password != null)
                password = Library.Utility.Utility.ByteArrayAsHexString(System.Security.Cryptography.SHA256.Create().ComputeHash(System.Text.Encoding.UTF8.GetBytes(password + "!" + uri.Scheme + "!" + uri.HostAndPath)));
                
            //Now find the one that matches :)
            var matches = (from n in configs
                where 
                    n.Type == type && 
                    //n.Passwordhash == password && 
                    n.Username == username && 
                    n.Port == port && 
                    n.Server == server && 
                    n.Path == path && 
                    n.Prefix == prefix
                select n).ToList();
            
            if (matches.Count > 1)
                throw new Exception(string.Format("Multiple sources found for: {0}", backend));
            
            // Re-select
            if (matches.Count == 0 && anyUsername && string.IsNullOrEmpty(username))
            {
                matches = (from n in configs
                    where 
                        n.Type == type && 
                        n.Port == port && 
                        n.Server == server && 
                        n.Path == path && 
                        n.Prefix == prefix
                    select n).ToList();
                    
                if (matches.Count > 1)
                    throw new Exception(String.Format("Multiple sources found for \"{0}\", try supplying --{1}", backend, "auth-username"));
            }
            
            if (matches.Count == 0 && !autoCreate)
                return null;
            
            if (matches.Count == 0)
            {
                var backupname = options.BackupName;
                if (string.IsNullOrEmpty(backupname) || backupname == Options.DefaultBackupName)
                    backupname = GenerateRandomName();
                else
                {
                    foreach(var c in System.IO.Path.GetInvalidFileNameChars())
                        backupname = backupname.Replace(c.ToString(), "");
                }
                
                var newpath = System.IO.Path.Combine(folder, backupname + ".sqlite");
                int max_tries = 100;
                while (System.IO.File.Exists(newpath) && max_tries-- > 0)
                    newpath = System.IO.Path.Combine(folder, GenerateRandomName());
                
                if (System.IO.File.Exists(newpath))
                    throw new Exception("Unable to find a unique name for the database, please use --dbpath");
                
                //Create a new one, add it to the list, and save it
                configs.Add(new BackendEntry() {
                    Type = type,
                    Server = server,
                    Path = path,
                    Prefix = prefix,
                    Username = username,
                    //Passwordhash = password,
                    Port = port,
                    Databasepath = newpath, 
                    ParameterFile = null
                });
                
                var settings = new Newtonsoft.Json.JsonSerializerSettings();
                settings.Formatting = Newtonsoft.Json.Formatting.Indented;
                System.IO.File.WriteAllText(file, Newtonsoft.Json.JsonConvert.SerializeObject(configs, settings), System.Text.Encoding.UTF8);
                
                return newpath;
            }
            else
            {
                return matches[0].Databasepath;
            }
            
        }
Beispiel #6
0
        internal static void VerifyParameters(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
        {
            var newDict = new Dictionary<string, string>();
            newDict.Add("blocksize", options.Blocksize.ToString());
            newDict.Add("blockhash", options.BlockHashAlgorithm);
            newDict.Add("filehash", options.FileHashAlgorithm);
            var opts = db.GetDbOptions(transaction);

            if (options.NoEncryption)
            {
                newDict.Add("passphrase", "no-encryption");
            }
            else
            {
                string salt;
                opts.TryGetValue("passphrase-salt", out salt);
                if (string.IsNullOrEmpty(salt))
                {
                    // Not Crypto-class PRNG salts
                    var buf = new byte[32];
                    new Random().NextBytes(buf);
                    //Add version so we can detect and change the algorithm
                    salt = "v1:" + Library.Utility.Utility.ByteArrayAsHexString(buf);
                }

                newDict["passphrase-salt"] = salt;

                // We avoid storing the passphrase directly,
                // instead we salt and rehash repeatedly
                newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200)));
            }

            var needsUpdate = false;
            foreach(var k in newDict)
                if (!opts.ContainsKey(k.Key))
                    needsUpdate = true;
                else if (opts[k.Key] != k.Value)
                {
                    if (k.Key == "passphrase")
                    {
                        if (!options.AllowPassphraseChange)
                        {
                            if (newDict[k.Key] == "no-encryption")
                                throw new Exception("Unsupported removal of passphrase");
                            else if (opts[k.Key] == "no-encryption")
                                throw new Exception("Unsupported addition of passphrase");
                            else
                                throw new Exception("Unsupported change of passphrase");
                        }
                    }
                    else
                        throw new Exception(string.Format("Unsupported change of parameter \"{0}\" from \"{1}\" to \"{2}\"", k.Key, opts[k.Key], k.Value));

                }

            //Extra sanity check
            if (db.GetBlocksLargerThan(options.Blocksize) > 0)
                throw new Exception("Unsupported block-size change detected");

            if (needsUpdate)
                db.SetDbOptions(newDict, transaction);
        }
Beispiel #7
0
 /// <summary>
 /// Constructs a container for a given metadata dictionary
 /// </summary>
 /// <param name="values">The metadata values to wrap</param>
 /// <returns>A IMetahash instance</returns>
 public static IMetahash WrapMetadata(Dictionary<string, string> values, Options options)
 {
     return new Metahash(values, options);
 }
Beispiel #8
0
            public bool UpdateHashAndSize(Options options)
            {
                if (Hash == null || Size < 0)
                {
                    Hash = CalculateFileHash(this.LocalFilename);
                    Size = new System.IO.FileInfo(this.LocalFilename).Length;
                    return true;
                }

                return false;
            }
Beispiel #9
0
        /// <summary>
        /// Constructs a new BackendWrapper
        /// </summary>
        /// <param name="statistics">The statistics logging module, may be null</param>
        /// <param name="backend">The url to the backend to wrap</param>
        /// <param name="options">A set of backend options</param>
        public BackendWrapper(CommunicationStatistics statistics, string backend, Options options)
        {
            m_statistics = statistics;
            m_options = options;

            m_filenamestrategy = new FilenameStrategy(m_options);
            m_backendUrl = backend;

            m_backend = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(backend, m_options.RawOptions);
            if (m_backend == null)
                throw new Exception(string.Format(Strings.BackendWrapper.BackendNotFoundError, backend));

            m_reuse_backend = !m_options.NoConnectionReuse;
            m_first_backend_use = true;
            m_backendSupportsCreateFolder = m_backend is Library.Interface.IBackend_v2;

            if (m_options.AutoCleanup)
                m_orphans = new List<BackupEntryBase>();

            if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && !System.IO.Directory.Exists(m_options.SignatureCachePath))
                System.IO.Directory.CreateDirectory(m_options.SignatureCachePath);

            if (!string.IsNullOrEmpty(m_options.Backendlogdatabase))
            {
                m_backendInterfaceLogger = new StateVerification.StateDatabase(m_options.Backendlogdatabase, statistics);
                m_backendInterfaceLogger.BeginOperation(m_options.MainAction.ToString());
            }

            m_async = m_options.AsynchronousUpload;
            if (m_async)
            {
                //If we are using async operations, the entire class is actually threadsafe,
                //utilizing a common exclusive lock on all operations. But the implementation does
                //not prevent starvation, so it should not be called by multiple threads.
                m_pendingOperations = new Queue<KeyValuePair<BackupEntryBase, string>>();
                m_asyncItemProcessed = new System.Threading.AutoResetEvent(false);
                m_asyncItemReady = new System.Threading.AutoResetEvent(false);
                m_workerThread = new System.Threading.Thread(ProcessQueue);
                m_workerThread.Name = "AsyncUploaderThread";
                m_queuelock = new object();
                m_workerThread.Start();
            }
        }
Beispiel #10
0
        public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
        {
            if (args.Count != 4)
            {
                Console.WriteLine("Invalid argument count ({0} expected 4): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
                return 100;
            }

            string target_compr_module = args[1];

            if (!Library.DynamicLoader.CompressionLoader.Keys.Contains(target_compr_module))
            {
                Console.WriteLine("Target compression module not found: {0}{1}Modules supported: {2}", args[1], Environment.NewLine, string.Join(", ", Library.DynamicLoader.CompressionLoader.Keys));
                return 100;
            }

            var m_Options = new Options(options);

            using (var backend = Library.DynamicLoader.BackendLoader.GetBackend(args[2], options))
            {
                if (backend == null)
                {
                    Console.WriteLine("Backend not found: {0}{1}Backends supported: {2}", args[2], Environment.NewLine, string.Join(", ", Library.DynamicLoader.BackendLoader.Keys));
                    return 100;
                }

                var targetfolder = Path.GetFullPath(args[3]);

                if (!Directory.Exists(args[3]))
                {
                    Console.WriteLine("Creating target folder: {0}", targetfolder);
                    Directory.CreateDirectory(targetfolder);
                }

                Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey);

                var rawlist = backend.List();

                Console.WriteLine("Found {0} files", rawlist.Count);

                var i = 0;
                var downloaded = 0;
                var errors = 0;
                var needspass = 0;

                var remotefiles =
                    (from x in rawlist
                     let n = VolumeBase.ParseFilename(x)
                     where n != null && n.Prefix == m_Options.Prefix
                     select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times

                if (remotefiles.Length == 0)
                {
                    if (rawlist.Count == 0)
                        Console.WriteLine("No files were found at the remote location, perhaps the target url is incorrect?");
                    else
                    {
                        var tmp =
                            (from x in rawlist
                             let n = VolumeBase.ParseFilename(x)
                             where
                                 n != null
                             select n.Prefix).ToArray();

                        var types = tmp.Distinct().ToArray();
                        if (tmp.Length == 0)
                            Console.WriteLine("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count);
                        else if (types.Length == 1)
                            Console.WriteLine("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0]);
                        else
                            Console.WriteLine("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types));
                    }

                    return 100;
                }

                bool reencrypt = Library.Utility.Utility.ParseBoolOption(options, "reencrypt");
                bool reupload = Library.Utility.Utility.ParseBoolOption(options, "reupload");

                foreach (var entry in remotefiles)
                {
                    try
                    {
                        Console.Write("{0}: {1}", i, entry.File.Name);

                        var local = Path.Combine(targetfolder, entry.File.Name);
                        if (entry.EncryptionModule != null)
                        {
                            if (string.IsNullOrWhiteSpace(m_Options.Passphrase))
                            {
                                needspass++;
                                Console.WriteLine(" - No passphrase supplied, skipping");
                                continue;
                            }

                            local = local.Substring(0, local.Length - entry.EncryptionModule.Length - 1);
                        }

                        if (entry.CompressionModule == target_compr_module)
                        {
                            Console.WriteLine(" - compression types are same");
                            continue;
                        }

                        string localNew;

                        if (entry.CompressionModule != null)
                        {
                            localNew = local.Substring(0, local.Length - entry.CompressionModule.Length - 1) + "." + target_compr_module;

                            if (File.Exists(localNew))
                            {
                                Console.WriteLine(" - target file already exist");
                                continue;
                            }
                        }
                        else
                        {
                            Console.WriteLine(" - cannot detect compression type");
                            continue;
                        }

                        if (File.Exists(local))
                            File.Delete(local);

                        Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(entry.File.Size));

                        DateTime originLastWriteTime;
                        FileInfo destinationFileInfo;

                        using (var tf = new Library.Utility.TempFile())
                        {
                            backend.Get(entry.File.Name, tf);
                            originLastWriteTime = new FileInfo(tf).LastWriteTime;
                            downloaded++;

                            if (entry.EncryptionModule != null)
                            {
                                Console.Write(" decrypting ...");
                                using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options))
                                using (var tf2 = new Library.Utility.TempFile())
                                {
                                    m.Decrypt(tf, tf2);
                                    File.Copy(tf2, local);
                                    File.Delete(tf2);
                                }
                            }
                            else
                                File.Copy(tf, local);

                            File.Delete(tf);
                            destinationFileInfo = new FileInfo(local);
                            destinationFileInfo.LastWriteTime = originLastWriteTime;
                        }

                        if (entry.CompressionModule != null)
                        {
                            Console.Write(" recompressing ...");

                            using (var cmOld = Library.DynamicLoader.CompressionLoader.GetModule(entry.CompressionModule, local, options))
                            using (var cmNew = Library.DynamicLoader.CompressionLoader.GetModule(target_compr_module, localNew, options))
                                foreach (var cmfile in cmOld.ListFiles(""))
                                {
                                    string cmfileNew = cmfile;

                                    if (entry.FileType == RemoteVolumeType.Index)
                                    {
                                        var cmFileVolume = Library.Main.Volumes.VolumeBase.ParseFilename(cmfileNew);
                                        if (cmFileVolume != null)
                                        {
                                            cmfileNew = cmfileNew.Replace("." + cmFileVolume.CompressionModule, "." + target_compr_module);
                                            if(!reencrypt)
                                                cmfileNew = cmfileNew.Replace("." + cmFileVolume.EncryptionModule, "");
                                        }
                                    }

                                    using (var sourceStream = cmOld.OpenRead(cmfile))
                                        using (var cs = cmNew.CreateFile(cmfileNew, Duplicati.Library.Interface.CompressionHint.Compressible, cmOld.GetLastWriteTime(cmfile)))
                                            Library.Utility.Utility.CopyStream(sourceStream, cs);
                                }

                            File.Delete(local);
                            destinationFileInfo = new FileInfo(localNew);
                            destinationFileInfo.LastWriteTime = originLastWriteTime;
                        }

                        if (reencrypt && entry.EncryptionModule != null)
                        {
                            Console.Write(" reencrypting ...");
                            using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options))
                            {
                                m.Encrypt(localNew, localNew + "." + m.FilenameExtension);
                                File.Delete(localNew);
                                localNew = localNew + "." + m.FilenameExtension;
                            }

                            destinationFileInfo = new FileInfo(localNew);
                            destinationFileInfo.LastWriteTime = originLastWriteTime;
                        }

                        if (reupload)
                        {
                            backend.Put((new FileInfo(localNew)).Name, localNew);
                            backend.Delete(entry.File.Name);
                            File.Delete(localNew);
                        }

                        Console.WriteLine(" done!");

                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine(" error: {0}", ex.ToString());
                        errors++;
                    }

                    i++;
                }

                if (reupload)
                {
                    var remoteverificationfileexist = rawlist.Any(x => x.Name == (m_Options.Prefix + "-verification.json"));

                    if (remoteverificationfileexist)
                    {
                        Console.WriteLine("Found verification file {0} - deleting", m_Options.Prefix + "-verification.json");
                        backend.Delete(m_Options.Prefix + "-verification.json");
                    }
                }

                if (needspass > 0 && downloaded == 0)
                {
                    Console.WriteLine("No files downloaded, try adding --passphrase to decrypt files");
                    return 100;
                }

                Console.WriteLine("Download complete, of {0} remote files, {1} were downloaded with {2} errors", remotefiles.Count(), downloaded, errors);
                if (needspass > 0)
                    Console.WriteLine("Additonally {0} remote files were skipped because of encryption, supply --passphrase to download those");

                if (errors > 0)
                {
                    Console.WriteLine("There were errors during recompress of remote backend files!");
                    return 200;
                }

                return 0;
            }
        }
Beispiel #11
0
 public static void PurgeSignatureCache(Dictionary<string, string> options)
 {
     Options opt = new Options(options);
     if (string.IsNullOrEmpty(opt.SignatureCachePath))
         throw new Exception(Strings.Interface.SignatureCachePathMissingError);
     else
         System.IO.Directory.Delete(opt.SignatureCachePath, true);
 }
Beispiel #12
0
 /// <summary>
 /// Constructs a new interface for performing backup and restore operations
 /// </summary>
 /// <param name="backend">The url for the backend to use</param>
 /// <param name="options">All required options</param>
 public Interface(string backend, Dictionary<string, string> options)
 {
     m_backend = backend;
     m_options = new Options(options);
     m_liveControl = new LiveControl.LiveControl(System.Threading.Thread.CurrentThread, m_options);
     OperationProgress += new OperationProgressEvent(Interface_OperationProgress);
 }