public async Task ListWithoutLocalDb()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb",
["no-local-db"] = "true"
};
// Run a full backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
}
}
public void Recover(string buildIndexWithFiles)
{
// Files to create in MB.
int[] fileSizes = { 10, 20, 30 };
foreach (int size in fileSizes)
{
byte[] data = new byte[size * 1024 * 1024];
Random rng = new Random();
rng.NextBytes(data);
File.WriteAllBytes(Path.Combine(this.DATAFOLDER, size + "MB"), data);
}
const string subdirectoryName = "subdirectory";
string subdirectoryPath = Path.Combine(this.DATAFOLDER, subdirectoryName);
Directory.CreateDirectory(subdirectoryPath);
foreach (int size in fileSizes)
{
byte[] data = new byte[size * 1024 * 1024];
Random rng = new Random();
rng.NextBytes(data);
File.WriteAllBytes(Path.Combine(subdirectoryPath, size + "MB"), data);
}
// Run a backup.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
string backendURL = "file://" + this.TARGETFOLDER;
using (Controller c = new Controller(backendURL, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Download the backend files.
string downloadFolder = Path.Combine(this.RESTOREFOLDER, "downloadedFiles");
Directory.CreateDirectory(downloadFolder);
int status = CommandLine.RecoveryTool.Program.RealMain(new[] { "download", $"{backendURL}", $"{downloadFolder}", $"--passphrase={options["passphrase"]}" });
Assert.AreEqual(0, status);
// Create the index.
status = CommandLine.RecoveryTool.Program.RealMain(new[] { "index", $"{downloadFolder}", $"--build-index-with-files={buildIndexWithFiles}" });
Assert.AreEqual(0, status);
// Restore to a different folder.
string restoreFolder = Path.Combine(this.RESTOREFOLDER, "restoredFiles");
Directory.CreateDirectory(restoreFolder);
status = CommandLine.RecoveryTool.Program.RealMain(new[] { "restore", $"{downloadFolder}", $"--targetpath={restoreFolder}" });
Assert.AreEqual(0, status);
// Since this.DATAFOLDER is a folder, Path.GetFileName will return the name of the
// last folder in the path.
string baseFolder = Path.GetFileName(this.DATAFOLDER);
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, Path.Combine(restoreFolder, baseFolder), false, "Verifying restore using RecoveryTool.");
}
public void RepairMissingIndexFiles(string noEncryption)
{
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["no-encryption"] = noEncryption
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
string[] dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray();
Assert.Greater(dindexFiles.Length, 0);
foreach (string f in dindexFiles)
{
File.Delete(f);
}
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
foreach (string file in dindexFiles)
{
Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file)));
}
}
public void CustomRemoteURL()
{
string customTargetFolder = Path.Combine(this.TARGETFOLDER, "destination");
Directory.CreateDirectory(customTargetFolder);
List <string> customCommands = new List <string>
{
$"echo --remoteurl = \"{customTargetFolder}\""
};
Dictionary <string, string> options = this.TestOptions;
options["run-script-before"] = CreateScript(0, null, null, 0, customCommands);
using (Controller c = new Library.Main.Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
string[] targetEntries = Directory.EnumerateFileSystemEntries(this.TARGETFOLDER).ToArray();
Assert.AreEqual(1, targetEntries.Length);
Assert.AreEqual(customTargetFolder, targetEntries[0]);
// We expect a dblock, dlist, and dindex file.
IEnumerable <string> customTargetEntries = Directory.EnumerateFileSystemEntries(customTargetFolder);
Assert.AreEqual(3, customTargetEntries.Count());
}
public void ExcludeProblematicPaths()
{
// A normal path that will be backed up.
string normalFilePath = Path.Combine(this.DATAFOLDER, "normal");
File.WriteAllBytes(normalFilePath, new byte[] { 0, 1, 2 });
// A long path to exclude.
string longFile = SystemIO.IO_OS.PathCombine(this.DATAFOLDER, new string('y', 255));
WriteFile(longFile, new byte[] { 0, 1 });
// A folder that ends with a dot to exclude.
string folderWithDot = Path.Combine(this.DATAFOLDER, "folder_with_dot.");
SystemIO.IO_OS.DirectoryCreate(folderWithDot);
// A folder that ends with a space to exclude.
string folderWithSpace = Path.Combine(this.DATAFOLDER, "folder_with_space ");
SystemIO.IO_OS.DirectoryCreate(folderWithSpace);
// A file that ends with a dot to exclude.
string fileWithDot = Path.Combine(this.DATAFOLDER, "file_with_dot.");
WriteFile(fileWithDot, new byte[] { 0, 1 });
// A file that ends with a space to exclude.
string fileWithSpace = Path.Combine(this.DATAFOLDER, "file_with_space ");
WriteFile(fileWithSpace, new byte[] { 0, 1 });
FilterExpression filter = new FilterExpression(longFile, false);
filter = FilterExpression.Combine(filter, new FilterExpression(Util.AppendDirSeparator(folderWithDot), false));
filter = FilterExpression.Combine(filter, new FilterExpression(Util.AppendDirSeparator(folderWithSpace), false));
filter = FilterExpression.Combine(filter, new FilterExpression(fileWithDot, false));
filter = FilterExpression.Combine(filter, new FilterExpression(fileWithSpace, false));
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IListResults listResults = c.List("*");
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
string[] backedUpPaths = listResults.Files.Select(x => x.Path).ToArray();
Assert.AreEqual(2, backedUpPaths.Length);
Assert.Contains(Util.AppendDirSeparator(this.DATAFOLDER), backedUpPaths);
Assert.Contains(normalFilePath, backedUpPaths);
}
}
public void ProblematicSuffixes(string pathComponent, bool skipOnWindows)
{
if (Platform.IsClientWindows && skipOnWindows)
{
return;
}
string folderPath = SystemIO.IO_OS.PathCombine(this.DATAFOLDER, pathComponent);
SystemIO.IO_OS.DirectoryCreate(folderPath);
string filePath = SystemIO.IO_OS.PathCombine(folderPath, pathComponent);
byte[] fileBytes = { 0, 1, 2 };
TestUtils.WriteFile(filePath, fileBytes);
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
{
["restore-path"] = this.RESTOREFOLDER
};
// Restore just the file.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
string restoreFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, pathComponent);
TestUtils.AssertFilesAreEqual(filePath, restoreFilePath, true, pathComponent);
SystemIO.IO_OS.FileDelete(restoreFilePath);
// Restore the entire directory.
string pathSpec = $"[{Regex.Escape(Util.AppendDirSeparator(this.DATAFOLDER))}.*]";
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(new[] { pathSpec });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, pathComponent);
}
public void RestoreEmptyFile()
{
string folderPath = Path.Combine(this.DATAFOLDER, "folder");
Directory.CreateDirectory(folderPath);
string filePath = Path.Combine(folderPath, "empty_file");
File.WriteAllBytes(filePath, new byte[] { });
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Issue #4148 described a situation where the folders containing the empty file were not recreated properly.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
{
["restore-path"] = this.RESTOREFOLDER,
["dont-compress-restore-paths"] = "true"
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
// We need to strip the root part of the path. Otherwise, Path.Combine will simply return the second argument
// if it's determined to be an absolute path.
string rootString = SystemIO.IO_OS.GetPathRoot(filePath);
string newPathPart = filePath.Substring(rootString.Length);
if (Platform.IsClientWindows)
{
// On Windows, the drive letter is included in the path when the dont-compress-restore-paths option is used.
// The drive letter is assumed to be the first character of the path root (e.g., C:\).
newPathPart = Path.Combine(rootString.Substring(0, 1), filePath.Substring(rootString.Length));
}
string restoredFilePath = Path.Combine(restoreOptions["restore-path"], newPathPart);
Assert.IsTrue(File.Exists(restoredFilePath));
}
public void LongPath()
{
string folderPath = Path.Combine(this.DATAFOLDER, new string('x', 10));
SystemIO.IO_OS.DirectoryCreate(folderPath);
string fileName = new string('y', 255);
string filePath = SystemIO.IO_OS.PathCombine(folderPath, fileName);
byte[] fileBytes = { 0, 1, 2 };
WriteFile(filePath, fileBytes);
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
}
string restoreFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName);
Assert.IsTrue(SystemIO.IO_OS.FileExists(restoreFilePath));
MemoryStream restoredStream = new MemoryStream();
using (FileStream fileStream = SystemIO.IO_OS.FileOpenRead(restoreFilePath))
{
Utility.CopyStream(fileStream, restoredStream);
}
Assert.AreEqual(fileBytes, restoredStream.ToArray());
}
public void RunScriptAfter(int exitCode)
{
const string expectedMessage = "Hello";
string expectedFile = Path.Combine(this.RESTOREFOLDER, "hello.txt");
List <string> customCommands = new List <string>
{
$"echo {expectedMessage}>\"{expectedFile}\""
};
Dictionary <string, string> options = this.TestOptions;
options["run-script-after"] = CreateScript(exitCode, null, null, 0, customCommands);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
switch (exitCode)
{
case 0:
case 1:
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
break;
case 2:
case 3:
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
break;
default:
Assert.AreEqual(1, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
break;
}
string[] targetEntries = Directory.EnumerateFileSystemEntries(this.RESTOREFOLDER).ToArray();
Assert.AreEqual(1, targetEntries.Length);
Assert.AreEqual(expectedFile, targetEntries[0]);
string[] lines = File.ReadAllLines(expectedFile);
Assert.AreEqual(1, lines.Length);
Assert.AreEqual(expectedMessage, lines[0]);
}
}
public async Task KeepVersionsRetention()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb"
};
// Run a full backup.
DateTime firstBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
firstBackupTime = c.List().Filesets.First().Time;
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
}
// Run a full backup.
DateTime fourthBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
this.ModifySourceFiles();
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
fourthBackupTime = c.List().Filesets.First().Time;
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
options["keep-versions"] = "2";
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
DateTime fifthBackupTime = c.List().Filesets.First().Time;
// Partial backups that are followed by a full backup can be deleted.
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(3, filesets.Count);
Assert.AreEqual(firstBackupTime, filesets[2].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(fifthBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
}
// Run a full backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
this.ModifySourceFiles();
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
DateTime sixthBackupTime = c.List().Filesets.First().Time;
// Since the last backup was full, we can now expect to have just the 2 most recent full backups.
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(sixthBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
}
public void DirectoriesWithWildcards()
{
const string file = "file";
List <string> directories = new List <string>();
// Keep expected match counts since they'll differ between
// Linux and Windows.
var questionMarkWildcardShouldMatchCount = 0;
var verbatimAsteriskShouldMatchCount = 0;
const string asterisk = "*";
string dirWithAsterisk = Path.Combine(this.DATAFOLDER, asterisk);
// Windows does not support literal asterisks in paths.
if (!Platform.IsClientWindows)
{
SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] { 0 });
directories.Add(dirWithAsterisk);
questionMarkWildcardShouldMatchCount++;
verbatimAsteriskShouldMatchCount++;
}
const string questionMark = "?";
string dirWithQuestionMark = Path.Combine(this.DATAFOLDER, questionMark);
// Windows does not support literal question marks in paths.
if (!Platform.IsClientWindows)
{
SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] { 1 });
directories.Add(dirWithQuestionMark);
questionMarkWildcardShouldMatchCount++;
}
// Include at least one single character directory in Windows
// for a '?' wildcard can match on
const string singleCharacterDir = "X";
string dirWithSingleCharacter = Path.Combine(this.DATAFOLDER, singleCharacterDir);
SystemIO.IO_OS.DirectoryCreate(dirWithSingleCharacter);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 2 });
directories.Add(dirWithSingleCharacter);
questionMarkWildcardShouldMatchCount++;
const string dir = "dir";
string normalDir = Path.Combine(this.DATAFOLDER, dir);
SystemIO.IO_OS.DirectoryCreate(normalDir);
WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] { 3 });
directories.Add(normalDir);
// Backup all files.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Restore all files.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(null);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
foreach (string directory in directories)
{
string directoryName = SystemIO.IO_OS.PathGetFileName(directory);
foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory))
{
string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, directoryName, fileName);
Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false));
}
}
// List results using * should return a match for each directory.
IListResults listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(directories.Count, listResults.Files.Count());
listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file));
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
// List results using ? should return 3 matches in Linux,
// one for the directory with '*' and one for the directory
// with '?', plus one for directory 'X'; but should return
// 1 matches in Windows just for directory 'X'.
Assert.AreEqual(questionMarkWildcardShouldMatchCount, listResults.Files.Count());
}
SystemIO.IO_OS.DirectoryDelete(this.RESTOREFOLDER, true);
// Restore one file at a time using the verbatim identifier.
foreach (string directory in directories)
{
foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory))
{
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
string verbatimFilePath = "@" + expectedFilePath;
// Verify that list result using verbatim identifier contains only one file.
IListResults listResults = c.List(verbatimFilePath);
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(1, listResults.Files.Count());
Assert.AreEqual(expectedFilePath, listResults.Files.Single().Path);
IRestoreResults restoreResults = c.Restore(new[] { verbatimFilePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName);
Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false));
SystemIO.IO_OS.FileDelete(restoredFilePath);
}
}
}
// Backup with asterisk in include filter should include all directories.
FilterExpression filter = new FilterExpression(dirWithAsterisk);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
Assert.AreEqual(directories.Count, backupResults.ExaminedFiles);
}
// Backup with verbatim asterisk in include filter should include
// one directory in Linux and zero directories in Windows.
filter = new FilterExpression("@" + SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }, filter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
Assert.AreEqual(verbatimAsteriskShouldMatchCount, backupResults.ExaminedFiles);
}
}
public void DeleteAllRemoteFiles()
{
string filePath = Path.Combine(this.DATAFOLDER, "file");
File.WriteAllBytes(filePath, new byte[] { 0, 1, 2 });
Dictionary <string, string> firstOptions = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, firstOptions, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Keep track of the backend files from the first backup configuration so that we can
// check that they remain after we remove the backend files from the second backup
// configuration.
string[] firstBackupFiles = Directory.GetFiles(this.TARGETFOLDER);
Assert.Greater(firstBackupFiles.Length, 0);
Dictionary <string, string> secondOptions = new Dictionary <string, string>(this.TestOptions)
{
["dbpath"] = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName())
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, secondOptions, null))
{
// An exception should be thrown due to unrecognized files in the target folder.
// ReSharper disable once AccessToDisposedClosure
Assert.That(() => c.Backup(new[] { this.DATAFOLDER }), Throws.Exception);
}
// We should be able to safely remove backend files from the second backup by referring
// to the local database.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, secondOptions, null))
{
IListRemoteResults listResults = c.DeleteAllRemoteFiles();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
}
// After we delete backend files from the second backup configuration, those from the first
// configuration should remain (see issues #3845, and #4244).
foreach (string file in firstBackupFiles)
{
Assert.IsTrue(File.Exists(file));
}
// Configure and run a second backup with a different prefix. This should run without error.
secondOptions["prefix"] = new Options(firstOptions).Prefix + "2";
using (Controller c = new Controller("file://" + this.TARGETFOLDER, secondOptions, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Even without a local database, we should be able to safely remove backend files from
// the second backup due to the prefix.
File.Delete(secondOptions["dbpath"]);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, secondOptions, null))
{
IListRemoteResults listResults = c.DeleteAllRemoteFiles();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
}
// After we delete backend files from the second backup configuration, those from the first
// configuration should remain (see issue #2678).
foreach (string file in firstBackupFiles)
{
Assert.IsTrue(File.Exists(file));
}
// The first backup configuration should still run normally.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, firstOptions, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
}
public async Task StopNow()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb", ["disable-synthetic-filelist"] = "true"
};
// Run a complete backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(1, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
// Interrupt a backup with "stop now".
this.ModifySourceFiles();
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
// ReSharper disable once AccessToDisposedClosure
Task backupTask = Task.Run(() => c.Backup(new[] { this.DATAFOLDER }));
// Block for a small amount of time to allow the ITaskControl to be associated
// with the Controller. Otherwise, the call to Stop will simply be a no-op.
Thread.Sleep(1000);
c.Stop(false);
await backupTask.ConfigureAwait(false);
}
// The next backup should proceed without issues.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
// Restore from the backup that followed the interruption.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
IRestoreResults restoreResults = c.Restore(fullVersionFiles);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
}
}
}
public async Task FilesetFiles()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb",
// This allows us to inspect the dlist files without needing the BackendManager (which is inaccessible here) to decrypt them.
["no-encryption"] = "true"
};
// Run a full backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
}
Dictionary <DateTime, int> GetBackupTypesFromRemoteFiles(Controller c, out List <string> filelistFiles)
{
Dictionary <DateTime, int> map = new Dictionary <DateTime, int>();
filelistFiles = new List <string>();
IListRemoteResults remoteFiles = c.ListRemote();
foreach (IFileEntry file in remoteFiles.Files)
{
IParsedVolume volume = VolumeBase.ParseFilename(file);
if (volume != null && volume.FileType == RemoteVolumeType.Files)
{
string dlistFile = Path.Combine(this.TARGETFOLDER, volume.File.Name);
filelistFiles.Add(dlistFile);
VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(volume.CompressionModule, dlistFile, new Options(options));
map[volume.Time] = filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP;
}
}
return(map);
}
// Purge a file and verify that the fileset file exists in the new dlist files.
List <string> dlistFiles;
Dictionary <DateTime, int> backupTypeMap;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IPurgeFilesResults purgeResults = c.PurgeFiles(new Library.Utility.FilterExpression($"*{this.fileSizes[0]}*"));
Assert.AreEqual(0, purgeResults.Errors.Count());
Assert.AreEqual(0, purgeResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets.Single(x => x.Version == 0).IsFullBackup);
backupTypeMap = GetBackupTypesFromRemoteFiles(c, out dlistFiles);
}
int[] backupTypes = backupTypeMap.OrderByDescending(x => x.Key).Select(x => x.Value).ToArray();
Assert.AreEqual(2, backupTypes.Length);
Assert.AreEqual(BackupType.FULL_BACKUP, backupTypes[1]);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, backupTypes[0]);
// Remove the dlist files.
foreach (string dlistFile in dlistFiles)
{
File.Delete(dlistFile);
}
// Run a repair and verify that the fileset file exists in the new dlist files.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets.Single(x => x.Version == 0).IsFullBackup);
backupTypeMap = GetBackupTypesFromRemoteFiles(c, out _);
}
backupTypes = backupTypeMap.OrderByDescending(x => x.Key).Select(x => x.Value).ToArray();
Assert.AreEqual(2, backupTypes.Length);
Assert.AreEqual(BackupType.FULL_BACKUP, backupTypes[1]);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, backupTypes[0]);
}
public async Task StopAfterCurrentFile()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb"
};
// Run a complete backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
Assert.AreEqual(1, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
// If we interrupt the backup, the most recent Fileset should be marked as partial.
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Restore files from the partial backup set.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(options)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] partialVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.GreaterOrEqual(partialVersionFiles.Length, 1);
c.Restore(partialVersionFiles);
foreach (string filepath in partialVersionFiles)
{
string filename = Path.GetFileName(filepath);
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
}
}
// Recreating the database should preserve the backup types.
File.Delete(this.DBFILE);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Run a complete backup. Listing the Filesets should include both full and partial backups.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
Assert.AreEqual(3, c.List().Filesets.Count());
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 2).IsFullBackup);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
}
// Restore files from the full backup set.
restoreOptions["overwrite"] = "true";
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IListResults lastResults = c.List("*");
string[] fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
IRestoreResults restoreResults = c.Restore(fullVersionFiles);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
}
}
}
public void RunScriptParsedResult(int exitCode)
{
string parsedResultFile = Path.Combine(this.RESTOREFOLDER, "result.txt");
List <string> customCommands = new List <string>();
if (Platform.IsClientWindows)
{
customCommands.Add($"echo %DUPLICATI__PARSED_RESULT%>\"{parsedResultFile}\"");
}
else
{
customCommands.Add($"echo $DUPLICATI__PARSED_RESULT>\"{parsedResultFile}\"");
}
Dictionary <string, string> options = this.TestOptions;
options["run-script-before"] = this.CreateScript(exitCode);
options["run-script-after"] = this.CreateScript(0, null, null, 0, customCommands);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
bool expectBackup;
string expectedParsedResult;
switch (exitCode)
{
case 0: // OK, run operation
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
expectBackup = true;
expectedParsedResult = ParsedResultType.Success.ToString();
break;
case 1: // OK, don't run operation
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
expectBackup = false;
expectedParsedResult = ParsedResultType.Success.ToString();
break;
case 2: // Warning, run operation
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
expectBackup = true;
expectedParsedResult = ParsedResultType.Warning.ToString();
break;
case 3: // Warning, don't run operation
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
expectBackup = false;
expectedParsedResult = ParsedResultType.Warning.ToString();
break;
case 4: // Error, run operation
Assert.AreEqual(1, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
expectBackup = true;
expectedParsedResult = ParsedResultType.Error.ToString();
break;
default: // Error don't run operation
Assert.AreEqual(1, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
expectBackup = false;
expectedParsedResult = ParsedResultType.Error.ToString();
break;
}
IEnumerable <string> targetEntries = Directory.EnumerateFileSystemEntries(this.TARGETFOLDER);
if (expectBackup)
{
// We expect a dblock, dlist, and dindex file.
Assert.AreEqual(3, targetEntries.Count());
}
else
{
Assert.AreEqual(0, targetEntries.Count());
}
string[] lines = File.ReadAllLines(parsedResultFile);
Assert.AreEqual(1, lines.Length);
Assert.AreEqual(expectedParsedResult, lines[0]);
}
}
private void RunCommands(int blocksize, int basedatasize = 0, Action <Dictionary <string, string> > modifyOptions = null)
{
var testopts = TestOptions;
testopts["blocksize"] = blocksize.ToString() + "b";
modifyOptions?.Invoke(testopts);
var filenames = WriteTestFilesToFolder(DATAFOLDER, blocksize, basedatasize);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
// TODO: This sometimes results in a "No block hash found for file: C:\projects\duplicati\testdata\backup-data\a-0" warning.
// Because of this, we don't check for warnings here.
}
// After the first backup we remove the --blocksize argument as that should be auto-set
testopts.Remove("blocksize");
testopts.Remove("block-hash-algorithm");
testopts.Remove("file-hash-algorithm");
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
IListResults listResults = c.List("*");
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
//Console.WriteLine("In first backup:");
//Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
}
// Do a "touch" on files to trigger a re-scan, which should do nothing
//foreach (var k in filenames)
//if (File.Exists(Path.Combine(DATAFOLDER, "a" + k.Key)))
//File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a" + k.Key), DateTime.Now.AddSeconds(5));
var data = new byte[filenames.Select(x => x.Value).Max()];
new Random().NextBytes(data);
foreach (var k in filenames)
{
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b" + k.Key), data.Take(k.Value).ToArray());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var r = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
if (!Library.Utility.Utility.ParseBoolOption(testopts, "disable-filetime-check"))
{
if (r.OpenedFiles != filenames.Count)
{
throw new Exception($"Opened {r.OpenedFiles}, but should open {filenames.Count}");
}
if (r.ExaminedFiles != filenames.Count * 2)
{
throw new Exception($"Examined {r.ExaminedFiles}, but should examine open {filenames.Count * 2}");
}
}
}
var rn = new Random();
foreach (var k in filenames)
{
rn.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "c" + k.Key), data.Take(k.Value).ToArray());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
//ProgressWriteLine("Newest before deleting:");
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0, no_local_db = true }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
//ProgressWriteLine("Newest without db:");
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count());
}
testopts["dbpath"] = this.recreatedDatabaseFile;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
// TODO: This sometimes results in a "No block hash found for file: C:\projects\duplicati\testdata\backup-data\a-0" warning.
// Because of this, we don't check for warnings here.
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(3, listResults.Filesets.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 2 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
//ProgressWriteLine("V2 after delete:");
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual((filenames.Count * 1) + 1, r.Files.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 1 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
//ProgressWriteLine("V1 after delete:");
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual((filenames.Count * 2) + 1, r.Files.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
//ProgressWriteLine("Newest after delete:");
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER, no_local_blocks = true }), null))
{
var r = c.Restore(null);
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Assert.AreEqual(filenames.Count * 3, r.RestoredFiles);
}
TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"));
using (var tf = new Library.Utility.TempFolder())
{
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = (string)tf, no_local_blocks = true }), null))
{
var r = c.Restore(new string[] { Path.Combine(DATAFOLDER, "a") + "*" });
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Assert.AreEqual(filenames.Count, r.RestoredFiles);
}
}
}
public async Task KeepTimeRetention()
{
// Choose a dblock size that is small enough so that more than one volume is needed.
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
["dblock-size"] = "10mb"
};
// First, run two complete backups followed by a partial backup. We will then set the keep-time
// option so that the threshold lies between the first and second backups.
DateTime firstBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
firstBackupTime = c.List().Filesets.First().Time;
}
// Wait before the second backup so that we can more easily define the keep-time threshold
// to lie between the first and second backups.
Thread.Sleep(5000);
DateTime secondBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
this.ModifySourceFiles();
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
secondBackupTime = c.List().Filesets.First().Time;
}
// Run a partial backup.
DateTime thirdBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
thirdBackupTime = c.List().Filesets.First().Time;
}
// Set the keep-time option so that the threshold lies between the first and second backups
// and run the delete operation.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
options["keep-time"] = $"{(int) ((DateTime.Now - firstBackupTime).TotalSeconds - (secondBackupTime - firstBackupTime).TotalSeconds / 2)}s";
IDeleteResults deleteResults = c.Delete();
Assert.AreEqual(0, deleteResults.Errors.Count());
Assert.AreEqual(0, deleteResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(secondBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(thirdBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
}
// Run another partial backup. We will then verify that a full backup is retained
// even when all the "recent" backups are partial.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
DateTime fourthBackupTime = c.List().Filesets.First().Time;
// Set the keep-time option so that the threshold lies after the most recent full backup
// and run the delete operation.
options["keep-time"] = "1s";
IDeleteResults deleteResults = c.Delete();
Assert.AreEqual(0, deleteResults.Errors.Count());
Assert.AreEqual(0, deleteResults.Warnings.Count());
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(3, filesets.Count);
Assert.AreEqual(secondBackupTime, filesets[2].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
Assert.AreEqual(thirdBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(fourthBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
}
}
public void RepairMissingBlocklistHashes()
{
byte[] data = new byte[150 * 1024];
Random rng = new Random();
for (int k = 0; k < 2; k++)
{
rng.NextBytes(data);
File.WriteAllBytes(Path.Combine(this.DATAFOLDER, $"{k}"), data);
}
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Mimic a damaged database that needs to be repaired.
const string selectStatement = @"SELECT BlocksetID, ""Index"", Hash FROM BlocklistHash ORDER BY Hash ASC";
List <int> expectedBlocksetIDs = new List <int>();
List <int> expectedIndexes = new List <int>();
List <string> expectedHashes = new List <string>();
using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"]))
{
// Read the contents of the BlocklistHash table so that we can
// compare them to the contents after the repair operation.
using (IDbCommand command = connection.CreateCommand())
{
using (IDataReader reader = command.ExecuteReader(selectStatement))
{
while (reader.Read())
{
expectedBlocksetIDs.Add(reader.GetInt32(0));
expectedIndexes.Add(reader.GetInt32(1));
expectedHashes.Add(reader.GetString(2));
}
}
}
using (IDbCommand command = connection.CreateCommand())
{
command.ExecuteNonQuery(@"DELETE FROM BlocklistHash");
using (IDataReader reader = command.ExecuteReader(selectStatement))
{
Assert.IsFalse(reader.Read());
}
}
}
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
List <int> repairedBlocksetIDs = new List <int>();
List <int> repairedIndexes = new List <int>();
List <string> repairedHashes = new List <string>();
using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"]))
{
using (IDbCommand command = connection.CreateCommand())
{
using (IDataReader reader = command.ExecuteReader(selectStatement))
{
while (reader.Read())
{
repairedBlocksetIDs.Add(reader.GetInt32(0));
repairedIndexes.Add(reader.GetInt32(1));
repairedHashes.Add(reader.GetString(2));
}
}
}
}
CollectionAssert.AreEqual(expectedBlocksetIDs, repairedBlocksetIDs);
CollectionAssert.AreEqual(expectedIndexes, repairedIndexes);
CollectionAssert.AreEqual(expectedHashes, repairedHashes);
// A subsequent backup should run without errors.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
}
public void RunCommands()
{
var testopts = TestOptions;
var data = new byte[1024 * 1024 * 10];
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Console.WriteLine("In first backup:");
Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
}
new Random().NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Console.WriteLine("Newest before deleting:");
Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual(3, r.Files.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0, no_local_db = true }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Console.WriteLine("Newest without db:");
Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual(3, r.Files.Count());
}
File.Delete(DBFILE);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(listResults.Filesets.Count(), 2);
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 1 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Console.WriteLine("Oldest after delete:");
Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual(2, r.Files.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Console.WriteLine("Newest after delete:");
Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
Assert.AreEqual(3, r.Files.Count());
}
}
public void TestEmptyFolderExclude()
{
var source = DATAFOLDER;
// Top level folder with no contents
Directory.CreateDirectory(Path.Combine(source, "empty-toplevel"));
// Top level folder with contents in one leaf
Directory.CreateDirectory(Path.Combine(source, "toplevel"));
// Empty folder
Directory.CreateDirectory(Path.Combine(source, "toplevel", "empty"));
// Folder with an excluded file
Directory.CreateDirectory(Path.Combine(source, "toplevel", "filteredempty"));
// Folder with contents
Directory.CreateDirectory(Path.Combine(source, "toplevel", "normal"));
// Folder with excludefile
Directory.CreateDirectory(Path.Combine(source, "toplevel", "excludefile"));
// Write a file that we will use for exclude target
File.WriteAllLines(Path.Combine(source, "toplevel", "excludefile", "exclude.me"), new string[] { });
File.WriteAllLines(Path.Combine(source, "toplevel", "excludefile", "anyfile.txt"), new string[] { "data" });
// Write a file that we will filter
File.WriteAllLines(Path.Combine(source, "toplevel", "filteredempty", "myfile.txt"), new string[] { "data" });
// Write a file that we will not filter
File.WriteAllLines(Path.Combine(source, "toplevel", "normal", "standard.txt"), new string[] { "data" });
// Get the default options
var testopts = TestOptions;
// Create a fileset with all data present
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Check that we have 4 files and 7 folders
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
var folders = r.Files.Count(x => x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
var files = r.Files.Count(x => !x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
if (folders != 7)
{
throw new Exception($"Initial condition not satisfied, found {folders} folders, but expected 7");
}
if (files != 4)
{
throw new Exception($"Initial condition not satisfied, found {files} files, but expected 4");
}
}
// Toggle the exclude file, and build a new fileset
System.Threading.Thread.Sleep(5000);
testopts["ignore-filenames"] = "exclude.me";
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Check that we have 2 files and 6 folders after excluding the "excludefile" folder
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
var folders = r.Files.Count(x => x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
var files = r.Files.Count(x => !x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
if (folders != 6)
{
throw new Exception($"Initial condition not satisfied, found {folders} folders, but expected 6");
}
if (files != 2)
{
throw new Exception($"Initial condition not satisfied, found {files} files, but expected 2");
}
}
// Toggle empty folder excludes, and run a new backup to remove them
System.Threading.Thread.Sleep(5000);
testopts["exclude-empty-folders"] = "true";
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Check that the two empty folders are now removed
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
var folders = r.Files.Count(x => x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
var files = r.Files.Count(x => !x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
if (folders != 4)
{
throw new Exception($"Empty not satisfied, found {folders} folders, but expected 4");
}
if (files != 2)
{
throw new Exception($"Empty not satisfied, found {files} files, but expected 2");
}
}
// Filter out one file and rerun the backup to exclude the folder
System.Threading.Thread.Sleep(5000);
var excludefilter = new Library.Utility.FilterExpression($"*{System.IO.Path.DirectorySeparatorChar}myfile.txt", false);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }, excludefilter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Check that the empty folder is now removed
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
var folders = r.Files.Count(x => x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
var files = r.Files.Count(x => !x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
if (folders != 3)
{
throw new Exception($"Empty not satisfied, found {folders} folders, but expected 3");
}
if (files != 1)
{
throw new Exception($"Empty not satisfied, found {files} files, but expected 1");
}
}
// Delete the one remaining file and check that we only have the top-level folder in the set
System.Threading.Thread.Sleep(5000);
File.Delete(Path.Combine(source, "toplevel", "normal", "standard.txt"));
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }, excludefilter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Check we now have only one folder and no files
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
{
var r = c.List("*");
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
var folders = r.Files.Count(x => x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
var files = r.Files.Count(x => !x.Path.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal));
if (folders != 1)
{
throw new Exception($"Empty not satisfied, found {folders} folders, but expected 1");
}
if (files != 0)
{
throw new Exception($"Empty not satisfied, found {files} files, but expected 0");
}
}
}
public void PurgeBrokenFilesTest()
{
var blocksize = 1024 * 10;
var basedatasize = 0;
var testopts = TestOptions;
testopts["blocksize"] = blocksize.ToString() + "b";
var filenames = BorderTests.WriteTestFilesToFolder(DATAFOLDER, blocksize, basedatasize).Select(x => "a" + x.Key).ToList();
var round1 = filenames.Take(filenames.Count / 3).ToArray();
var round2 = filenames.Take((filenames.Count / 3) * 2).ToArray();
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER }, new Library.Utility.FilterExpression(round1.Select(x => "*" + Path.DirectorySeparatorChar + x)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(res.AddedFiles, round1.Length);
}
var dblock_file = SystemIO.IO_OS
.GetFiles(TARGETFOLDER, "*.dblock.zip.aes")
.Select(x => new FileInfo(x))
.OrderBy(x => x.LastWriteTimeUtc)
.Select(x => x.FullName)
.First();
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER }, new Library.Utility.FilterExpression(round2.Select(x => "*" + Path.DirectorySeparatorChar + x)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(round2.Length - round1.Length, res.AddedFiles);
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(filenames.Count - round2.Length, res.AddedFiles);
}
File.Delete(dblock_file);
long[] affectedfiles;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var brk = c.ListBrokenFiles(null);
Assert.AreEqual(0, brk.Errors.Count());
Assert.AreEqual(0, brk.Warnings.Count());
var sets = brk.BrokenFiles.Count();
var files = brk.BrokenFiles.Sum(x => x.Item3.Count());
Assert.AreEqual(3, sets);
Assert.True(files > 0);
affectedfiles = brk.BrokenFiles.OrderBy(x => x.Item1).Select(x => x.Item3.LongCount()).ToArray();
}
for (var i = 0; i < 3; i++)
{
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = i }), null))
{
var brk = c.ListBrokenFiles(null);
Assert.AreEqual(0, brk.Errors.Count());
Assert.AreEqual(0, brk.Warnings.Count());
var sets = brk.BrokenFiles.Count();
var files = brk.BrokenFiles.Sum(x => x.Item3.Count());
Assert.AreEqual(1, sets);
Assert.AreEqual(affectedfiles[i], files);
}
}
// A dry-run should run without exceptions (see issue #4379).
Dictionary <string, string> dryRunOptions = new Dictionary <string, string>(testopts)
{
["dry-run"] = "true"
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, dryRunOptions, null))
{
IPurgeBrokenFilesResults purgeResults = c.PurgeBrokenFiles(null);
Assert.AreEqual(0, purgeResults.Errors.Count());
Assert.AreEqual(0, purgeResults.Warnings.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var brk = c.PurgeBrokenFiles(null);
Assert.AreEqual(0, brk.Errors.Count());
Assert.AreEqual(0, brk.Warnings.Count());
var modFilesets = 0L;
if (brk.DeleteResults != null)
{
modFilesets += brk.DeleteResults.DeletedSets.Count();
}
if (brk.PurgeResults != null)
{
modFilesets += brk.PurgeResults.RewrittenFileLists;
}
Assert.AreEqual(3, modFilesets);
}
// A subsequent backup should be successful.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
}
public void SymLinkPolicy(Options.SymlinkStrategy symlinkPolicy)
{
// Create symlink target directory
const string targetDirName = "target";
var targetDir = systemIO.PathCombine(this.DATAFOLDER, targetDirName);
systemIO.DirectoryCreate(targetDir);
// Create files in symlink target directory
var fileNames = new[] { "a.txt", "b.txt", "c.txt" };
foreach (var file in fileNames)
{
var targetFile = systemIO.PathCombine(targetDir, file);
TestUtils.WriteFile(targetFile, Encoding.Default.GetBytes(file));
}
// Create actual symlink directory linking to the target directory
const string symlinkDirName = "symlink";
var symlinkDir = systemIO.PathCombine(this.DATAFOLDER, symlinkDirName);
try
{
systemIO.CreateSymlink(symlinkDir, targetDir, asDir: true);
}
catch (Exception e)
{
// If client cannot create symlinks, mark test as ignored
Assert.Ignore($"Client could not create a symbolic link. Error reported: {e.Message}");
}
// Backup all files with given symlink policy
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
{
["restore-path"] = this.RESTOREFOLDER
};
Dictionary <string, string> backupOptions = new Dictionary <string, string>(this.TestOptions)
{
["symlink-policy"] = symlinkPolicy.ToString()
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Restore all files
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(null);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
// Verify that symlink policy was followed
var restoreSymlinkDir = systemIO.PathCombine(this.RESTOREFOLDER, symlinkDirName);
switch (symlinkPolicy)
{
case Options.SymlinkStrategy.Store:
// Restore should contain an actual symlink to the original target
Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.True);
var restoredSymlinkFullPath = systemIO.PathGetFullPath(systemIO.GetSymlinkTarget(restoreSymlinkDir));
var symlinkTargetFullPath = systemIO.PathGetFullPath(targetDir);
Assert.That(restoredSymlinkFullPath, Is.EqualTo(symlinkTargetFullPath));
break;
case Options.SymlinkStrategy.Follow:
// Restore should contain a regular directory with copies of the files in the symlink target
Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.False);
TestUtils.AssertDirectoryTreesAreEquivalent(targetDir, restoreSymlinkDir, true, "Restore");
break;
case Options.SymlinkStrategy.Ignore:
// Restore should not contain the symlink or directory at all
Assert.That(systemIO.DirectoryExists(restoreSymlinkDir), Is.False);
Assert.That(systemIO.FileExists(restoreSymlinkDir), Is.False);
break;
default:
Assert.Fail($"Unexpected symlink policy");
break;
}
}
}
public void RestoreInheritanceBreaks()
{
if (!Platform.IsClientWindows)
{
return;
}
string folderPath = Path.Combine(this.DATAFOLDER, "folder");
Directory.CreateDirectory(folderPath);
string filePath = Path.Combine(folderPath, "file");
File.WriteAllBytes(filePath, new byte[] { 0 });
// Protect access rules on the file.
FileSecurity fileSecurity = File.GetAccessControl(filePath);
fileSecurity.SetAccessRuleProtection(true, true);
File.SetAccessControl(filePath, fileSecurity);
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// First, restore without restoring permissions.
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
{
["restore-path"] = this.RESTOREFOLDER
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
string restoredFilePath = Path.Combine(this.RESTOREFOLDER, "file");
Assert.IsTrue(File.Exists(restoredFilePath));
FileSecurity restoredFileSecurity = File.GetAccessControl(restoredFilePath);
Assert.IsFalse(restoredFileSecurity.AreAccessRulesProtected);
// Remove the restored file so that the later restore avoids the "Restore completed
// without errors but no files were restored" warning.
File.Delete(restoredFilePath);
}
// Restore with restoring permissions.
restoreOptions["overwrite"] = "true";
restoreOptions["restore-permissions"] = "true";
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(new[] { filePath });
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
string restoredFilePath = Path.Combine(this.RESTOREFOLDER, "file");
Assert.IsTrue(File.Exists(restoredFilePath));
FileSecurity restoredFileSecurity = File.GetAccessControl(restoredFilePath);
Assert.IsTrue(restoredFileSecurity.AreAccessRulesProtected);
}
}
public void PurgeTest()
{
var blocksize = 1024 * 10;
var basedatasize = 0;
var testopts = TestOptions;
testopts["blocksize"] = blocksize.ToString() + "b";
var filenames = BorderTests.WriteTestFilesToFolder(DATAFOLDER, blocksize, basedatasize).Select(x => "a" + x.Key).ToList();
var round1 = filenames.Take(filenames.Count / 3).ToArray();
var round2 = filenames.Take((filenames.Count / 3) * 2).ToArray();
var round3 = filenames;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER }, new Library.Utility.FilterExpression(round1.Select(x => "*" + Path.DirectorySeparatorChar + x)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(res.AddedFiles, round1.Length);
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER }, new Library.Utility.FilterExpression(round2.Select(x => "*" + Path.DirectorySeparatorChar + x)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(res.AddedFiles, round2.Length - round1.Length);
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(res.AddedFiles, filenames.Count - round2.Length);
}
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
var last_ts = DateTime.Now;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { list_sets_only = true }), null))
{
var inf = c.List();
Assert.AreEqual(0, inf.Errors.Count());
Assert.AreEqual(0, inf.Warnings.Count());
var filesets = inf.Filesets.Count();
Assert.AreEqual(3, filesets, "Incorrect number of initial filesets");
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List("*");
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
var filecount = listResults.Files.Count();
Assert.AreEqual(filenames.Count + 1, filecount, "Incorrect number of initial files");
}
var allversion_candidate = round1.First();
var single_version_candidate = round1.Skip(1).First();
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.PurgeFiles(new Library.Utility.FilterExpression("*" + Path.DirectorySeparatorChar + allversion_candidate));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(3, res.RewrittenFileLists, "Incorrect number of rewritten filesets after all-versions purge");
Assert.AreEqual(3, res.RemovedFileCount, "Incorrect number of removed files after all-versions purge");
}
for (var i = 0; i < 3; i++)
{
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = i }), null))
{
var res = c.PurgeFiles(new Library.Utility.FilterExpression(Path.Combine(this.DATAFOLDER, single_version_candidate)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(1, res.RewrittenFileLists, "Incorrect number of rewritten filesets after single-versions purge");
Assert.AreEqual(1, res.RemovedFileCount, "Incorrect number of removed files after single-versions purge");
}
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.PurgeFiles(new Library.Utility.FilterExpression(round2.Skip(round1.Length).Take(2).Select(x => "*" + Path.DirectorySeparatorChar + x)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(2, res.RewrittenFileLists, "Incorrect number of rewritten filesets after 2-versions purge");
Assert.AreEqual(4, res.RemovedFileCount, "Incorrect number of removed files after 2-versions purge");
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.PurgeFiles(new Library.Utility.FilterExpression(round3.Skip(round2.Length).Take(2).Select(x => "*" + Path.DirectorySeparatorChar + x)));
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
Assert.AreEqual(1, res.RewrittenFileLists, "Incorrect number of rewritten filesets after 1-versions purge");
Assert.AreEqual(2, res.RemovedFileCount, "Incorrect number of removed files after 1-versions purge");
}
// Since we make the operations back-to-back, the purge timestamp can drift beyond the current time
var wait_target = last_ts.AddSeconds(10) - DateTime.Now;
if (wait_target.TotalMilliseconds > 0)
{
System.Threading.Thread.Sleep(wait_target);
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var listinfo = c.List("*");
Assert.AreEqual(0, listinfo.Errors.Count());
Assert.AreEqual(0, listinfo.Warnings.Count());
var filecount = listinfo.Files.Count();
listinfo = c.List();
Assert.AreEqual(0, listinfo.Errors.Count());
Assert.AreEqual(0, listinfo.Warnings.Count());
var filesets = listinfo.Filesets.Count();
Assert.AreEqual(3, filesets, "Incorrect number of filesets after purge");
Assert.AreEqual(filenames.Count - 6 + 1, filecount, "Incorrect number of files after purge");
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var listinfo = c.List("*");
Assert.AreEqual(0, listinfo.Errors.Count());
Assert.AreEqual(0, listinfo.Warnings.Count());
var files = listinfo.Files.ToArray();
var filecount = files.Length;
listinfo = c.List();
Assert.AreEqual(0, listinfo.Errors.Count());
Assert.AreEqual(0, listinfo.Warnings.Count());
var filesets = listinfo.Filesets.ToArray();
Console.WriteLine("Listing final version information");
Console.WriteLine("Versions:");
Console.WriteLine(" " + string.Join(Environment.NewLine + " ", filesets.Select(x => string.Format("{0}: {1}, {2} {3}", x.Version, x.Time, x.FileCount, x.FileSizes))));
Console.WriteLine("Files:");
Console.WriteLine(" " + string.Join(Environment.NewLine + " ", files.Select(x => string.Format("{0}: {1}", x.Path, string.Join(" - ", x.Sizes.Select(y => y.ToString()))))));
Assert.AreEqual(4, filesets.Length, "Incorrect number of filesets after final backup");
Assert.AreEqual(filenames.Count + 1, filecount, "Incorrect number of files after final backup");
}
}
public async Task RetentionPolicyRetention()
{
Dictionary <string, string> options = new Dictionary <string, string>(this.TestOptions)
{
// Choose a dblock size that is small enough so that more than one volume is needed.
["dblock-size"] = "10mb",
// This test assumes that we can perform 3 backups within 1 minute.
["retention-policy"] = "1m:59s,U:1m",
["no-backend-verification"] = "true"
};
DateTime firstBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
firstBackupTime = c.List().Filesets.First().Time;
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(1, filesets.Count);
this.ModifySourceFiles();
backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
DateTime secondBackupTime = c.List().Filesets.First().Time;
// Since the most recent backup is not considered in the retention logic, the only backup in the first time frame
// is the initial one. As a result, we should have 2 backups.
filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(firstBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(secondBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
// Wait so that the next backups fall in the next retention interval.
Thread.Sleep(new TimeSpan(0, 0, 1, 0));
DateTime thirdBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
thirdBackupTime = c.List().Filesets.First().Time;
// Since the most recent backup is not considered in the retention logic, there are no backups in the first time
// frame. The original 2 backups have now spilled over to the U:1m specification. Since we keep the first
// backup in the interval, we should be left with the first backup, as well as the third partial one.
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(2, filesets.Count);
Assert.AreEqual(firstBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(thirdBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
}
DateTime fourthBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
this.ModifySourceFiles();
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
fourthBackupTime = c.List().Filesets.First().Time;
// Since the most recent backup is not considered in the retention logic, the third backup is the only backup
// in the first time frame. There is no further spillover, so we simply add the fourth backup to the
// collection of retained backups.
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(3, filesets.Count);
Assert.AreEqual(firstBackupTime, filesets[2].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
Assert.AreEqual(thirdBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(fourthBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
this.ModifySourceFiles();
backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
DateTime fifthBackupTime = c.List().Filesets.First().Time;
// Since the most recent backup is not considered in the retention logic, we now have two backups in the
// first time frame: the third (partial) and fourth (full). Since the first backup in each interval is
// kept, we would typically keep just the third backup. However, since we should not discard a full
// backup in favor of a partial one, we keep the fourth as well. We also still have the initial backup.
filesets = c.List().Filesets.ToList();
Assert.AreEqual(4, filesets.Count);
Assert.AreEqual(firstBackupTime, filesets[3].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup);
Assert.AreEqual(thirdBackupTime, filesets[2].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup);
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(fifthBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
// Wait so that the next backups fall in the next retention interval.
Thread.Sleep(new TimeSpan(0, 0, 1, 0));
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
this.ModifySourceFiles();
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
DateTime sixthBackupTime = c.List().Filesets.First().Time;
// Since the most recent backup is not considered in the retention logic, we now have three backups in the
// second time frame: the third (partial), fourth (full), and fifth (full). Since we keep up to the first
// full backup in each time frame, we now drop the fifth backup.
List <IListResultFileset> filesets = c.List().Filesets.ToList();
Assert.AreEqual(4, filesets.Count);
Assert.AreEqual(firstBackupTime, filesets[3].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup);
Assert.AreEqual(thirdBackupTime, filesets[2].Time);
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup);
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
Assert.AreEqual(sixthBackupTime, filesets[0].Time);
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
}
}
