private static void VerifyPartialRestore(string source, IEnumerable <string> testfiles, string[] actualfolders, string tempfolder, string rootfolder, bool verifymetadata)
{
using (new Timer(LOGTAG, "PartialRestoreVerify", "Verification of partial restore from " + source))
foreach (string s in testfiles)
{
string restoredname;
string sourcename;
if (actualfolders.Length == 1)
{
sourcename = System.IO.Path.Combine(actualfolders[0], s);
restoredname = System.IO.Path.Combine(tempfolder, s);
}
else
{
int six = s.IndexOf(System.IO.Path.DirectorySeparatorChar);
sourcename = System.IO.Path.Combine(actualfolders[int.Parse(s.Substring(0, six))], s.Substring(six + 1));
restoredname = System.IO.Path.Combine(System.IO.Path.Combine(tempfolder, System.IO.Path.GetFileName(rootfolder.Split(System.IO.Path.PathSeparator)[int.Parse(s.Substring(0, six))])), s.Substring(six + 1));
}
if (!System.IO.File.Exists(restoredname))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", restoredname);
BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + restoredname);
}
else
{
if (!System.IO.File.Exists(sourcename))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreMissingFile", null, "Partial restore missing file: {0}", sourcename);
BasicSetupHelper.ProgressWriteLine("Partial restore missing file: " + sourcename);
throw new Exception("Unittest is broken");
}
if (!TestUtils.CompareFiles(sourcename, restoredname, s, verifymetadata))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreWrongFile", null, "Partial restore file differs: {0}", s);
BasicSetupHelper.ProgressWriteLine("Partial restore file differs: " + s);
}
}
}
}
public void Recover(string buildIndexWithFiles)
{
// Files to create in MB.
int[] fileSizes = { 10, 20, 30 };
foreach (int size in fileSizes)
{
byte[] data = new byte[size * 1024 * 1024];
Random rng = new Random();
rng.NextBytes(data);
File.WriteAllBytes(Path.Combine(this.DATAFOLDER, size + "MB"), data);
}
const string subdirectoryName = "subdirectory";
string subdirectoryPath = Path.Combine(this.DATAFOLDER, subdirectoryName);
Directory.CreateDirectory(subdirectoryPath);
foreach (int size in fileSizes)
{
byte[] data = new byte[size * 1024 * 1024];
Random rng = new Random();
rng.NextBytes(data);
File.WriteAllBytes(Path.Combine(subdirectoryPath, size + "MB"), data);
}
// Run a backup.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
string backendURL = "file://" + this.TARGETFOLDER;
using (Controller c = new Controller(backendURL, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
}
// Download the backend files.
string downloadFolder = Path.Combine(this.RESTOREFOLDER, "downloadedFiles");
Directory.CreateDirectory(downloadFolder);
int status = CommandLine.RecoveryTool.Program.RealMain(new[] { "download", $"{backendURL}", $"{downloadFolder}", $"--passphrase={options["passphrase"]}" });
Assert.AreEqual(0, status);
// Create the index.
status = CommandLine.RecoveryTool.Program.RealMain(new[] { "index", $"{downloadFolder}", $"--build-index-with-files={buildIndexWithFiles}" });
Assert.AreEqual(0, status);
// Restore to a different folder.
string restoreFolder = Path.Combine(this.RESTOREFOLDER, "restoredFiles");
Directory.CreateDirectory(restoreFolder);
status = CommandLine.RecoveryTool.Program.RealMain(new[] { "restore", $"{downloadFolder}", $"--targetpath={restoreFolder}" });
Assert.AreEqual(0, status);
// Since this.DATAFOLDER is a folder, Path.GetFileName will return the name of the
// last folder in the path.
string baseFolder = Path.GetFileName(this.DATAFOLDER);
foreach (string filepath in Directory.EnumerateFiles(this.DATAFOLDER))
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(restoreFolder, baseFolder, filename ?? String.Empty), filename, false));
}
foreach (string filepath in Directory.EnumerateFiles(subdirectoryPath))
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(restoreFolder, baseFolder, subdirectoryName, filename ?? String.Empty), filename, false));
}
}
public void DirectoriesWithWildcards()
{
const string file = "file";
List <string> directories = new List <string>();
// Keep expected match counts since they'll differ between
// Linux and Windows.
var questionMarkWildcardShouldMatchCount = 0;
var verbatimAsteriskShouldMatchCount = 0;
const string asterisk = "*";
string dirWithAsterisk = Path.Combine(this.DATAFOLDER, asterisk);
// Windows does not support literal asterisks in paths.
if (!Platform.IsClientWindows)
{
SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] { 0 });
directories.Add(dirWithAsterisk);
questionMarkWildcardShouldMatchCount++;
verbatimAsteriskShouldMatchCount++;
}
const string questionMark = "?";
string dirWithQuestionMark = Path.Combine(this.DATAFOLDER, questionMark);
// Windows does not support literal question marks in paths.
if (!Platform.IsClientWindows)
{
SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] { 1 });
directories.Add(dirWithQuestionMark);
questionMarkWildcardShouldMatchCount++;
}
// Include at least one single character directory in Windows
// for a '?' wildcard can match on
const string singleCharacterDir = "X";
string dirWithSingleCharacter = Path.Combine(this.DATAFOLDER, singleCharacterDir);
SystemIO.IO_OS.DirectoryCreate(dirWithSingleCharacter);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 2 });
directories.Add(dirWithSingleCharacter);
questionMarkWildcardShouldMatchCount++;
const string dir = "dir";
string normalDir = Path.Combine(this.DATAFOLDER, dir);
SystemIO.IO_OS.DirectoryCreate(normalDir);
WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] { 3 });
directories.Add(normalDir);
// Backup all files.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Restore all files.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(null);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
foreach (string directory in directories)
{
string directoryName = SystemIO.IO_OS.PathGetFileName(directory);
foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory))
{
string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, directoryName, fileName);
Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false));
}
}
// List results using * should return a match for each directory.
IListResults listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(directories.Count, listResults.Files.Count());
listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file));
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
// List results using ? should return 3 matches in Linux,
// one for the directory with '*' and one for the directory
// with '?', plus one for directory 'X'; but should return
// 1 matches in Windows just for directory 'X'.
Assert.AreEqual(questionMarkWildcardShouldMatchCount, listResults.Files.Count());
}
SystemIO.IO_OS.DirectoryDelete(this.RESTOREFOLDER, true);
// Restore one file at a time using the verbatim identifier.
foreach (string directory in directories)
{
foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory))
{
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
string verbatimFilePath = "@" + expectedFilePath;
// Verify that list result using verbatim identifier contains only one file.
IListResults listResults = c.List(verbatimFilePath);
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(1, listResults.Files.Count());
Assert.AreEqual(expectedFilePath, listResults.Files.Single().Path);
IRestoreResults restoreResults = c.Restore(new[] { verbatimFilePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName);
Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false));
SystemIO.IO_OS.FileDelete(restoredFilePath);
}
}
}
// Backup with asterisk in include filter should include all directories.
FilterExpression filter = new FilterExpression(dirWithAsterisk);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
Assert.AreEqual(directories.Count, backupResults.ExaminedFiles);
}
// Backup with verbatim asterisk in include filter should include
// one directory in Linux and zero directories in Windows.
filter = new FilterExpression("@" + SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
Assert.AreEqual(verbatimAsteriskShouldMatchCount, backupResults.ExaminedFiles);
}
}
public async Task StopAfterCurrentFile()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb"
};
// Run a complete backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(1, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
await this.RunPartialBackup(c).ConfigureAwait(false);
// If we interrupt the backup, the most recent Fileset should be marked as partial.
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Restore files from the partial backup set.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] partialVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.GreaterOrEqual(partialVersionFiles.Length, 1);
c.Restore(partialVersionFiles);
foreach (string filepath in partialVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
}
}
// Recreating the database should preserve the backup types.
File.Delete(this.DBFILE);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Repair();
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Run a complete backup. Listing the Filesets should include both full and partial backups.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(3, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 2).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Restore files from the full backup set.
restoreOptions["overwrite"] = "true";
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
c.Restore(fullVersionFiles);
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
}
}
}
public async Task StopNow()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb", ["disable-synthetic-filelist"] = "true"
};
// Run a complete backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(1, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
// Interrupt a backup with "stop now".
this.ModifySourceFiles();
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
// ReSharper disable once AccessToDisposedClosure
Task backupTask = Task.Run(() => c.Backup(new[] { this.DATAFOLDER }));
// Block for a small amount of time to allow the ITaskControl to be associated
// with the Controller. Otherwise, the call to Stop will simply be a no-op.
Thread.Sleep(1000);
c.Stop(false);
await backupTask.ConfigureAwait(false);
}
// The next backup should proceed without issues.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
// Restore from the backup that followed the interruption.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
c.Restore(fullVersionFiles);
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
}
}
}
public async Task StopAfterCurrentFile()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb"
};
// Run a complete backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(1, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Modify the source files and interrupt a backup.
this.ModifySourceFiles();
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
// ReSharper disable once AccessToDisposedClosure
Task <IBackupResults> backupTask = Task.Run(() => c.Backup(new[] { this.DATAFOLDER }));
// Block for a small amount of time to allow the ITaskControl to be associated
// with the Controller. Otherwise, the call to Stop will simply be a no-op.
Thread.Sleep(1000);
// If we interrupt the backup, the most recent Fileset should be marked as partial.
c.Stop(true);
await backupTask.ConfigureAwait(false);
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Restore files from the partial backup set.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] partialVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.GreaterOrEqual(partialVersionFiles.Length, 1);
c.Restore(partialVersionFiles);
foreach (string filepath in partialVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
}
}
// Recreating the database should preserve the backup types.
File.Delete(this.DBFILE);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Repair();
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Run a complete backup. Listing the Filesets should omit the previous partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Restore files from the full backup set.
restoreOptions["overwrite"] = "true";
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
c.Restore(fullVersionFiles);
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
}
}
}