/// <summary> /// Running the unit test confirms the correctness of duplicati /// </summary> /// <param name="folders">The folders to backup. Folder at index 0 is the base, all others are incrementals</param> /// <param name="target">The target destination for the backups</param> public static void RunTest(string[] folders, Dictionary<string, string> options, string target) { LogHelper log = new LogHelper("unittest.log"); Log.CurrentLog = log; ; Log.LogLevel = Duplicati.Library.Logging.LogMessageType.Profiling; string tempdir = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "tempdir"); //Filter empty entries, commonly occuring with copy/paste and newlines folders = (from x in folders where !string.IsNullOrWhiteSpace(x) select Library.Utility.Utility.ExpandEnvironmentVariables(x)).ToArray(); //Expand the tilde to home folder on Linux/OSX if (Utility.IsClientLinux) folders = (from x in folders select x.Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.Personal))).ToArray(); foreach(var f in folders) foreach(var n in f.Split(new char[] {System.IO.Path.PathSeparator}, StringSplitOptions.RemoveEmptyEntries)) if (!System.IO.Directory.Exists(n)) throw new Exception(string.Format("Missing source folder: {0}", n)); try { if (System.IO.Directory.Exists(tempdir)) System.IO.Directory.Delete(tempdir, true); System.IO.Directory.CreateDirectory(tempdir); } catch(Exception ex) { Log.WriteMessage("Failed to clean tempdir", LogMessageType.Error, ex); } Duplicati.Library.Utility.TempFolder.SystemTempPath = tempdir; //Set some defaults if (!options.ContainsKey("passphrase")) options["passphrase"] = "secret password!"; if (!options.ContainsKey("prefix")) options["prefix"] = "duplicati_unittest"; //We want all messages in the log options["log-level"] = LogMessageType.Profiling.ToString(); //We cannot rely on USN numbering, but we can use USN enumeration //options["disable-usn-diff-check"] = "true"; //We use precise times options["disable-time-tolerance"] = "true"; //We need all sets, even if they are unchanged options["upload-unchanged-backups"] = "true"; bool skipfullrestore = false; bool skippartialrestore = false; bool skipverify = false; if (Utility.ParseBoolOption(options, "unittest-backuponly")) { skipfullrestore = true; skippartialrestore = true; options.Remove("unittest-backuponly"); } if (Utility.ParseBoolOption(options, "unittest-skip-partial-restore")) { skippartialrestore = true; options.Remove("unittest-skip-partial-restore"); } if (Utility.ParseBoolOption(options, "unittest-skip-full-restore")) { skipfullrestore = true; options.Remove("unittest-skip-full-restore"); } if (Utility.ParseBoolOption(options, "unittest-skip-verify")) { skipverify = true; options.Remove("unittest-skip-verify"); } var verifymetadata = !Utility.ParseBoolOption(options, "skip-metadata"); using(new Timer("Total unittest")) using(TempFolder tf = new TempFolder()) { options["dbpath"] = System.IO.Path.Combine(tempdir, "unittest.sqlite"); if (System.IO.File.Exists(options["dbpath"])) System.IO.File.Delete(options["dbpath"]); if (string.IsNullOrEmpty(target)) { target = "file://" + tf; } else { Console.WriteLine("Removing old backups"); Dictionary<string, string> tmp = new Dictionary<string, string>(options); tmp["keep-versions"] = "0"; tmp["force"] = ""; tmp["allow-full-removal"] = ""; using(new Timer("Cleaning up any existing backups")) try { using(var bk = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(target, options)) foreach(var f in bk.List()) if (!f.IsFolder) bk.Delete(f.Name); } catch(Duplicati.Library.Interface.FolderMissingException) { } } log.Backupset = "Backup " + folders[0]; string fhtempsource = null; bool usingFHWithRestore = (!skipfullrestore || !skippartialrestore); using(var fhsourcefolder = usingFHWithRestore ? new Library.Utility.TempFolder() : null) { if (usingFHWithRestore) { fhtempsource = fhsourcefolder; TestUtils.CopyDirectoryRecursive(folders[0], fhsourcefolder); } RunBackup(usingFHWithRestore ? (string)fhsourcefolder : folders[0], target, options, folders[0]); for (int i = 1; i < folders.Length; i++) { //options["passphrase"] = "bad password"; //If the backups are too close, we can't pick the right one :( System.Threading.Thread.Sleep(1000 * 5); log.Backupset = "Backup " + folders[i]; if (usingFHWithRestore) { System.IO.Directory.Delete(fhsourcefolder, true); TestUtils.CopyDirectoryRecursive(folders[i], fhsourcefolder); } //Call function to simplify profiling RunBackup(usingFHWithRestore ? (string)fhsourcefolder : folders[i], target, options, folders[i]); } } Duplicati.Library.Main.Options opts = new Duplicati.Library.Main.Options(options); using (Duplicati.Library.Interface.IBackend bk = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(target, options)) foreach (Duplicati.Library.Interface.IFileEntry fe in bk.List()) if (fe.Size > opts.VolumeSize) { string msg = string.Format("The file {0} is {1} bytes larger than allowed", fe.Name, fe.Size - opts.VolumeSize); Console.WriteLine(msg); Log.WriteMessage(msg, LogMessageType.Error); } IList<DateTime> entries; using(var i = new Duplicati.Library.Main.Controller(target, options, new CommandLine.ConsoleOutput(options))) entries = (from n in i.List().Filesets select n.Time.ToLocalTime()).ToList(); if (entries.Count != folders.Length) { StringBuilder sb = new StringBuilder(); sb.AppendLine("Entry count: " + entries.Count.ToString()); sb.Append(string.Format("Found {0} filelists but there were {1} source folders", entries.Count, folders.Length)); throw new Exception("Filename parsing problem, or corrupt storage: " + sb.ToString()); } if (!skipfullrestore || !skippartialrestore) { for (int i = 0; i < entries.Count; i++) { using (TempFolder ttf = new TempFolder()) { log.Backupset = "Restore " + folders[i]; Console.WriteLine("Restoring the copy: " + folders[i]); options["time"] = entries[entries.Count - i - 1].ToString(); string[] actualfolders = folders[i].Split(System.IO.Path.PathSeparator); if (!skippartialrestore) { Console.WriteLine("Partial restore of: " + folders[i]); using (TempFolder ptf = new TempFolder()) { List<string> testfiles = new List<string>(); using (new Timer("Extract list of files from" + folders[i])) { List<string> sourcefiles; using(var inst = new Library.Main.Controller(target, options, new CommandLine.ConsoleOutput(options))) sourcefiles = (from n in inst.List("*").Files select n.Path).ToList(); //Remove all folders from list for (int j = 0; j < sourcefiles.Count; j++) if (sourcefiles[j].EndsWith(System.IO.Path.DirectorySeparatorChar.ToString())) { sourcefiles.RemoveAt(j); j--; } int testfilecount = 15; Random r = new Random(); while (testfilecount-- > 0 && sourcefiles.Count > 0) { int rn = r.Next(0, sourcefiles.Count); testfiles.Add(sourcefiles[rn]); sourcefiles.RemoveAt(rn); } } //Add all folders to avoid warnings in restore log int c = testfiles.Count; Dictionary<string, string> partialFolders = new Dictionary<string, string>(Utility.ClientFilenameStringComparer); for (int j = 0; j < c; j++) { string f = testfiles[j]; if (!f.StartsWith(usingFHWithRestore ? fhtempsource : folders[i], Utility.ClientFilenameStringComparision)) throw new Exception(string.Format("Unexpected file found: {0}, path is not a subfolder for {1}", f, folders[i])); f = f.Substring(Utility.AppendDirSeparator(usingFHWithRestore ? fhtempsource : folders[i]).Length); do { f = System.IO.Path.GetDirectoryName(f); partialFolders[Utility.AppendDirSeparator(f)] = null; } while (f.IndexOf(System.IO.Path.DirectorySeparatorChar) > 0); } if (partialFolders.ContainsKey("")) partialFolders.Remove(""); if (partialFolders.ContainsKey(System.IO.Path.DirectorySeparatorChar.ToString())) partialFolders.Remove(System.IO.Path.DirectorySeparatorChar.ToString()); List<string> filterlist; var tfe = Utility.AppendDirSeparator(usingFHWithRestore ? fhtempsource : folders[i]); filterlist = (from n in partialFolders.Keys where !string.IsNullOrWhiteSpace(n) && n != System.IO.Path.DirectorySeparatorChar.ToString() select Utility.AppendDirSeparator(System.IO.Path.Combine(tfe, n))) .Union(testfiles) //Add files with full path .Union(new string[] { tfe }) //Ensure root folder is included .Distinct() .ToList(); testfiles = (from n in testfiles select n.Substring(tfe.Length)).ToList(); //Call function to simplify profiling RunPartialRestore(folders[i], target, ptf, options, filterlist.ToArray()); if (!skipverify) { //Call function to simplify profiling Console.WriteLine("Verifying partial restore of: " + folders[i]); VerifyPartialRestore(folders[i], testfiles, actualfolders, ptf, folders[0], verifymetadata); } } } if (!skipfullrestore) { //Call function to simplify profiling RunRestore(folders[i], target, ttf, options); if (!skipverify) { //Call function to simplify profiling Console.WriteLine("Verifying the copy: " + folders[i]); VerifyFullRestore(folders[i], actualfolders, new string[] { ttf }, verifymetadata); } } } } } foreach(string s in Utility.EnumerateFiles(tempdir)) { if (s == options["dbpath"]) continue; if (s.StartsWith(Utility.AppendDirSeparator(tf))) continue; Log.WriteMessage(string.Format("Found left-over temp file: {0}", s.Substring(tempdir.Length)), LogMessageType.Warning); Console.WriteLine("Found left-over temp file: {0} -> {1}", s.Substring(tempdir.Length), TempFile.GetStackTraceForTempFile(System.IO.Path.GetFileName(s))); } foreach(string s in Utility.EnumerateFolders(tempdir)) if (!s.StartsWith(Utility.AppendDirSeparator(tf)) && Utility.AppendDirSeparator(s) != Utility.AppendDirSeparator(tf) && Utility.AppendDirSeparator(s) != Utility.AppendDirSeparator(tempdir)) { Log.WriteMessage(string.Format("Found left-over temp folder: {0}", s.Substring(tempdir.Length)), LogMessageType.Warning); Console.WriteLine("Found left-over temp folder: {0}", s.Substring(tempdir.Length)); } } (Log.CurrentLog as StreamLog).Dispose(); Log.CurrentLog = null; if (LogHelper.ErrorCount > 0) Console.WriteLine("Unittest completed, but with {0} errors, see logfile for details", LogHelper.ErrorCount); else if (LogHelper.WarningCount > 0) Console.WriteLine("Unittest completed, but with {0} warnings, see logfile for details", LogHelper.WarningCount); else Console.WriteLine("Unittest completed successfully - Have some cake!"); }