Esempio n. 1
0
        static bool Run(List <string> args, Dictionary <string, string> options, bool first)
        {
            Library.Interface.IBackend backend = Library.DynamicLoader.BackendLoader.GetBackend(args[0], options);
            if (backend == null)
            {
                Console.WriteLine("Unsupported backend");
                Console.WriteLine();
                Console.WriteLine("Supported backends: " + string.Join(",", Duplicati.Library.DynamicLoader.BackendLoader.Keys));
                return(false);
            }

            string allowedChars = ValidFilenameChars;

            if (options.ContainsKey("extended-chars"))
            {
                allowedChars += String.IsNullOrEmpty(options["extended-chars"]) ? ExtendedChars : options["extended-chars"];
            }

            bool autoCreateFolders = Library.Utility.Utility.ParseBoolOption(options, "auto-create-folder");

            string disabledModulesValue;
            string enabledModulesValue;

            options.TryGetValue("enable-module", out enabledModulesValue);
            options.TryGetValue("disable-module", out disabledModulesValue);
            string[] enabledModules  = enabledModulesValue == null ? new string[0] : enabledModulesValue.Trim().ToLower(CultureInfo.InvariantCulture).Split(',');
            string[] disabledModules = disabledModulesValue == null ? new string[0] : disabledModulesValue.Trim().ToLower(CultureInfo.InvariantCulture).Split(',');

            List <Library.Interface.IGenericModule> loadedModules = new List <IGenericModule>();

            foreach (Library.Interface.IGenericModule m in Library.DynamicLoader.GenericLoader.Modules)
            {
                if (!disabledModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || enabledModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase)))
                {
                    m.Configure(options);
                    loadedModules.Add(m);
                }
            }

            try
            {
                IEnumerable <Library.Interface.IFileEntry> curlist = null;
                try
                {
                    backend.Test();
                    curlist = backend.List();
                }
                catch (FolderMissingException)
                {
                    if (autoCreateFolders)
                    {
                        try
                        {
                            backend.CreateFolder();
                            curlist = backend.List();
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine("Autocreate folder failed with message: " + ex.Message);
                        }
                    }

                    if (curlist == null)
                    {
                        throw;
                    }
                }

                foreach (Library.Interface.IFileEntry fe in curlist)
                {
                    if (!fe.IsFolder)
                    {
                        if (Library.Utility.Utility.ParseBoolOption(options, "auto-clean") && first)
                        {
                            if (Library.Utility.Utility.ParseBoolOption(options, "force"))
                            {
                                Console.WriteLine("Auto clean, removing file: {0}", fe.Name);
                                backend.Delete(fe.Name);
                                continue;
                            }
                            else
                            {
                                Console.WriteLine("Specify the --force flag to actually delete files");
                            }
                        }

                        Console.WriteLine("*** Remote folder is not empty, aborting");
                        return(false);
                    }
                }


                int  number_of_files    = 10;
                int  min_file_size      = 1024;
                int  max_file_size      = 1024 * 1024 * 50;
                int  min_filename_size  = 5;
                int  max_filename_size  = 80;
                bool disableStreaming   = Library.Utility.Utility.ParseBoolOption(options, "disable-streaming-transfers");
                bool skipOverwriteTest  = Library.Utility.Utility.ParseBoolOption(options, "skip-overwrite-test");
                bool trimFilenameSpaces = Library.Utility.Utility.ParseBoolOption(options, "trim-filename-spaces");

                if (options.ContainsKey("number-of-files"))
                {
                    number_of_files = int.Parse(options["number-of-files"]);
                }
                if (options.ContainsKey("min-file-size"))
                {
                    min_file_size = (int)Duplicati.Library.Utility.Sizeparser.ParseSize(options["min-file-size"], "mb");
                }
                if (options.ContainsKey("max-file-size"))
                {
                    max_file_size = (int)Duplicati.Library.Utility.Sizeparser.ParseSize(options["max-file-size"], "mb");
                }

                if (options.ContainsKey("min-filename-length"))
                {
                    min_filename_size = int.Parse(options["min-filename-length"]);
                }
                if (options.ContainsKey("max-filename-length"))
                {
                    max_filename_size = int.Parse(options["max-filename-length"]);
                }

                Random rnd = new Random();
                System.Security.Cryptography.SHA256 sha = System.Security.Cryptography.SHA256.Create();

                //Create random files
                using (Library.Utility.TempFolder tf = new Duplicati.Library.Utility.TempFolder())
                {
                    List <TempFile> files = new List <TempFile>();
                    for (int i = 0; i < number_of_files; i++)
                    {
                        string filename = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd);

                        string localfilename = CreateRandomFile(tf, i, min_file_size, max_file_size, rnd);

                        //Calculate local hash and length
                        using (System.IO.FileStream fs = new System.IO.FileStream(localfilename, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                            files.Add(new TempFile(filename, localfilename, sha.ComputeHash(fs), fs.Length));
                    }

                    byte[] dummyFileHash = null;
                    if (!skipOverwriteTest)
                    {
                        Console.WriteLine("Uploading wrong files ...");
                        using (Library.Utility.TempFile dummy = Library.Utility.TempFile.WrapExistingFile(CreateRandomFile(tf, files.Count, 1024, 2048, rnd)))
                        {
                            using (System.IO.FileStream fs = new System.IO.FileStream(dummy, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                                dummyFileHash = sha.ComputeHash(fs);

                            //Upload a dummy file for entry 0 and the last one, they will be replaced by the real files afterwards
                            //We upload entry 0 twice just to try to freak any internal cache list
                            Uploadfile(dummy, 0, files[0].remotefilename, backend, disableStreaming);
                            Uploadfile(dummy, 0, files[0].remotefilename, backend, disableStreaming);
                            Uploadfile(dummy, files.Count - 1, files[files.Count - 1].remotefilename, backend, disableStreaming);
                        }
                    }

                    Console.WriteLine("Uploading files ...");

                    for (int i = 0; i < files.Count; i++)
                    {
                        Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming);
                    }

                    TempFile originalRenamedFile = null;
                    string   renamedFileNewName  = null;
                    IRenameEnabledBackend renameEnabledBackend = backend as IRenameEnabledBackend;
                    if (renameEnabledBackend != null)
                    {
                        // Rename the second file in the list, if there are more than one. If not, just do the first one.
                        int renameIndex = files.Count > 1 ? 1 : 0;
                        originalRenamedFile = files[renameIndex];

                        renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd);

                        Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName);

                        renameEnabledBackend.Rename(originalRenamedFile.remotefilename, renamedFileNewName);
                        files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length);
                    }

                    Console.WriteLine("Verifying file list ...");

                    curlist = backend.List();
                    foreach (Library.Interface.IFileEntry fe in curlist)
                    {
                        if (!fe.IsFolder)
                        {
                            bool found = false;
                            foreach (TempFile tx in files)
                            {
                                if (tx.remotefilename == fe.Name)
                                {
                                    if (tx.found)
                                    {
                                        Console.WriteLine("*** File with name {0} was found more than once", tx.remotefilename);
                                    }
                                    found    = true;
                                    tx.found = true;

                                    if (fe.Size > 0 && tx.length != fe.Size)
                                    {
                                        Console.WriteLine("*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size);
                                    }

                                    break;
                                }
                            }

                            if (!found)
                            {
                                if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name)
                                {
                                    Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName);
                                }
                                else
                                {
                                    Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name);
                                }
                            }
                        }
                    }

                    foreach (TempFile tx in files)
                    {
                        if (!tx.found)
                        {
                            Console.WriteLine("*** File with name {0} was uploaded but not found afterwards", tx.remotefilename);
                        }
                    }

                    Console.WriteLine("Downloading files");

                    for (int i = 0; i < files.Count; i++)
                    {
                        using (Duplicati.Library.Utility.TempFile cf = new Duplicati.Library.Utility.TempFile())
                        {
                            Exception e = null;
                            Console.Write("Downloading file {0} ... ", i);

                            try
                            {
                                if (backend is IStreamingBackend streamingBackend && !disableStreaming)
                                {
                                    using (System.IO.FileStream fs = new System.IO.FileStream(cf, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
                                        using (NonSeekableStream nss = new NonSeekableStream(fs))
                                            streamingBackend.Get(files[i].remotefilename, nss);
                                }
                                else
                                {
                                    backend.Get(files[i].remotefilename, cf);
                                }

                                e = null;
                            }
                            catch (Exception ex)
                            {
                                e = ex;
                            }

                            if (e != null)
                            {
                                Console.WriteLine("failed\n*** Error: {0}", e);
                            }
                            else
                            {
                                Console.WriteLine("done");
                            }

                            Console.Write("Checking hash ... ");

                            using (System.IO.FileStream fs = new System.IO.FileStream(cf, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                                if (Convert.ToBase64String(sha.ComputeHash(fs)) != Convert.ToBase64String(files[i].hash))
                                {
                                    if (dummyFileHash != null && Convert.ToBase64String(sha.ComputeHash(fs)) == Convert.ToBase64String(dummyFileHash))
                                    {
                                        Console.WriteLine("failed\n*** Downloaded file was the dummy file");
                                    }
                                    else
                                    {
                                        Console.WriteLine("failed\n*** Downloaded file was corrupt");
                                    }
                                }
                                else
                                {
                                    Console.WriteLine("done");
                                }
                        }
Esempio n. 2
0
        static bool Run(List <string> args, Dictionary <string, string> options, bool first)
        {
            string allowedChars = ValidFilenameChars;

            if (options.ContainsKey("extended-chars"))
            {
                allowedChars += options["extended-chars"];
            }
            else
            {
                allowedChars += ExtendedChars;
            }

            bool autoCreateFolders = Library.Utility.Utility.ParseBoolOption(options, "auto-create-folder");

            Library.Interface.IBackend backend = Library.DynamicLoader.BackendLoader.GetBackend(args[0], options);
            if (backend == null)
            {
                Console.WriteLine("Unsupported backend");
                Console.WriteLine();
                Console.WriteLine("Supported backends: " + string.Join(",", Duplicati.Library.DynamicLoader.BackendLoader.Keys));
                return(false);
            }

            string disabledModulesValue;
            string enabledModulesValue;

            options.TryGetValue("enable-module", out enabledModulesValue);
            options.TryGetValue("disable-module", out disabledModulesValue);
            string[] enabledModules  = enabledModulesValue == null ? new string[0] : enabledModulesValue.Trim().ToLower().Split(',');
            string[] disabledModules = disabledModulesValue == null ? new string[0] : disabledModulesValue.Trim().ToLower().Split(',');

            List <Library.Interface.IGenericModule> loadedModules = new List <IGenericModule>();

            foreach (Library.Interface.IGenericModule m in Library.DynamicLoader.GenericLoader.Modules)
            {
                if (Array.IndexOf <string>(disabledModules, m.Key.ToLower()) < 0 && (m.LoadAsDefault || Array.IndexOf <string>(enabledModules, m.Key.ToLower()) >= 0))
                {
                    m.Configure(options);
                    loadedModules.Add(m);
                }
            }

            try
            {
                List <Library.Interface.IFileEntry> curlist = null;
                try
                {
                    curlist = backend.List();
                }
                catch (FolderMissingException fex)
                {
                    if (autoCreateFolders)
                    {
                        try
                        {
                            if (backend is IBackend_v2)
                            {
                                ((IBackend_v2)backend).CreateFolder();
                            }

                            curlist = backend.List();
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine("Autocreate folder failed with message: " + ex.Message);
                        }
                    }

                    if (curlist == null)
                    {
                        throw fex;
                    }
                }

                foreach (Library.Interface.IFileEntry fe in curlist)
                {
                    if (!fe.IsFolder)
                    {
                        if (Library.Utility.Utility.ParseBoolOption(options, "auto-clean") && first)
                        {
                            if (Library.Utility.Utility.ParseBoolOption(options, "force"))
                            {
                                Console.WriteLine("Auto clean, removing file: {0}", fe.Name);
                                backend.Delete(fe.Name);
                                continue;
                            }
                            else
                            {
                                Console.WriteLine("Specify the --force flag to actually delete files");
                            }
                        }

                        Console.WriteLine("*** Remote folder is not empty, aborting");
                        return(false);
                    }
                }


                int  number_of_files   = 10;
                int  min_file_size     = 1024;
                int  max_file_size     = 1024 * 1024 * 50;
                int  min_filename_size = 5;
                int  max_filename_size = 80;
                bool disableStreaming  = Library.Utility.Utility.ParseBoolOption(options, "disable-streaming-transfers");
                bool skipOverwriteTest = Library.Utility.Utility.ParseBoolOption(options, "skip-overwrite-test");

                if (options.ContainsKey("number-of-files"))
                {
                    number_of_files = int.Parse(options["number-of-files"]);
                }
                if (options.ContainsKey("min-file-size"))
                {
                    min_file_size = (int)Duplicati.Library.Utility.Sizeparser.ParseSize(options["min-file-size"], "mb");
                }
                if (options.ContainsKey("max-file-size"))
                {
                    max_file_size = (int)Duplicati.Library.Utility.Sizeparser.ParseSize(options["max-file-size"]);
                }

                if (options.ContainsKey("min-filename-length"))
                {
                    min_filename_size = int.Parse(options["min-filename-length"]);
                }
                if (options.ContainsKey("max-filename-length"))
                {
                    max_filename_size = int.Parse(options["max-filename-length"]);
                }

                Random rnd = new Random();
                System.Security.Cryptography.SHA256 sha = System.Security.Cryptography.SHA256.Create();

                //Create random files
                using (Library.Utility.TempFolder tf = new Duplicati.Library.Utility.TempFolder())
                {
                    List <TempFile> files = new List <TempFile>();
                    for (int i = 0; i < number_of_files; i++)
                    {
                        StringBuilder filename    = new StringBuilder();
                        int           filenamelen = rnd.Next(min_filename_size, max_filename_size);
                        for (int j = 0; j < filenamelen; j++)
                        {
                            filename.Append(allowedChars[rnd.Next(0, allowedChars.Length)]);
                        }

                        string localfilename = CreateRandomFile(tf, i, min_file_size, max_file_size, rnd);

                        //Calculate local hash and length
                        using (System.IO.FileStream fs = new System.IO.FileStream(localfilename, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                            files.Add(new TempFile(filename.ToString(), localfilename, sha.ComputeHash(fs), fs.Length));
                    }

                    byte[] dummyFileHash = null;
                    if (!skipOverwriteTest)
                    {
                        Console.WriteLine("Uploading wrong files ...");
                        using (Library.Utility.TempFile dummy = new Library.Utility.TempFile(CreateRandomFile(tf, files.Count, 1024, 2048, rnd)))
                        {
                            using (System.IO.FileStream fs = new System.IO.FileStream(dummy, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                                dummyFileHash = sha.ComputeHash(fs);

                            //Upload a dummy file for entry 0 and the last one, they will be replaced by the real files afterwards
                            //We upload entry 0 twice just to try to freak any internal cache list
                            Uploadfile(dummy, 0, files[0].remotefilename, backend, disableStreaming);
                            Uploadfile(dummy, 0, files[0].remotefilename, backend, disableStreaming);
                            Uploadfile(dummy, files.Count - 1, files[files.Count - 1].remotefilename, backend, disableStreaming);
                        }
                    }

                    Console.WriteLine("Uploading files ...");

                    for (int i = 0; i < files.Count; i++)
                    {
                        Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming);
                    }

                    Console.WriteLine("Verifying file list ...");

                    curlist = backend.List();
                    foreach (Library.Interface.IFileEntry fe in curlist)
                    {
                        if (!fe.IsFolder)
                        {
                            bool found = false;
                            foreach (TempFile tx in files)
                            {
                                if (tx.remotefilename == fe.Name)
                                {
                                    if (tx.found)
                                    {
                                        Console.WriteLine("*** File with name {0} was found more than once", tx.remotefilename);
                                    }
                                    found    = true;
                                    tx.found = true;

                                    if (fe.Size > 0 && tx.length != fe.Size)
                                    {
                                        Console.WriteLine("*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size);
                                    }

                                    break;
                                }
                            }

                            if (!found)
                            {
                                Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name);
                            }
                        }
                    }

                    foreach (TempFile tx in files)
                    {
                        if (!tx.found)
                        {
                            Console.WriteLine("*** File with name {0} was uploaded but not found afterwards", tx.remotefilename);
                        }
                    }

                    Console.WriteLine("Downloading files");

                    for (int i = 0; i < files.Count; i++)
                    {
                        using (Duplicati.Library.Utility.TempFile cf = new Duplicati.Library.Utility.TempFile())
                        {
                            Exception e = null;
                            Console.Write("Downloading file {0} ... ", i);

                            try
                            {
                                if (backend is Library.Interface.IStreamingBackend && !disableStreaming)
                                {
                                    using (System.IO.FileStream fs = new System.IO.FileStream(cf, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
                                        using (NonSeekableStream nss = new NonSeekableStream(fs))
                                            (backend as Library.Interface.IStreamingBackend).Get(files[i].remotefilename, nss);
                                }
                                else
                                {
                                    backend.Get(files[i].remotefilename, cf);
                                }

                                e = null;
                            }
                            catch (Exception ex)
                            {
                                e = ex;
                            }

                            if (e != null)
                            {
                                Console.WriteLine("failed\n*** Error: {0}", e.ToString());
                            }
                            else
                            {
                                Console.WriteLine("done");
                            }

                            Console.Write("Checking hash ... ");

                            using (System.IO.FileStream fs = new System.IO.FileStream(cf, System.IO.FileMode.Open, System.IO.FileAccess.Read))
                                if (Convert.ToBase64String(sha.ComputeHash(fs)) != Convert.ToBase64String(files[i].hash))
                                {
                                    if (dummyFileHash != null && Convert.ToBase64String(sha.ComputeHash(fs)) == Convert.ToBase64String(dummyFileHash))
                                    {
                                        Console.WriteLine("failed\n*** Downloaded file was the dummy file");
                                    }
                                    else
                                    {
                                        Console.WriteLine("failed\n*** Downloaded file was corrupt");
                                    }
                                }
                                else
                                {
                                    Console.WriteLine("done");
                                }
                        }
                    }

                    Console.WriteLine("Deleting files...");

                    foreach (TempFile tx in files)
                    {
                        try { backend.Delete(tx.remotefilename); }
                        catch (Exception ex)
                        {
                            Console.WriteLine("*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex.ToString());
                        }
                    }

                    curlist = backend.List();
                    foreach (Library.Interface.IFileEntry fe in curlist)
                    {
                        if (!fe.IsFolder)
                        {
                            Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name);
                        }
                    }
                }
            }
            finally
            {
                foreach (Library.Interface.IGenericModule m in loadedModules)
                {
                    if (m is IDisposable)
                    {
                        ((IDisposable)m).Dispose();
                    }
                }
            }

            return(true);
        }