private void EnsureFileSystemHasRequiredSettings(string id, FileSystemDocument fsDoc)
{
if (!fsDoc.Settings.ContainsKey(Constants.FileSystem.DataDirectory))
{
fsDoc.Settings[Constants.FileSystem.DataDirectory] = "~/FileSystems/" + id;
}
}
public async Task CreateFileSystemWhenExistingWillFail()
{
var client = NewAsyncClient();
var adminClient = client.Admin;
const string newFileSystemName = "testName_CreateFileSystemWhenExistingWillFail";
var fileSystemSpec = new FileSystemDocument
{
Id = "Raven/FileSystem/" + newFileSystemName,
Settings =
{
{ "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", newFileSystemName)) }
}
};
await adminClient.CreateFileSystemAsync(fileSystemSpec, newFileSystemName);
var names = await adminClient.GetNamesAsync();
Assert.Contains(newFileSystemName, names);
bool throwsException = false;
try
{
await adminClient.CreateFileSystemAsync(fileSystemSpec, newFileSystemName);
}
catch (InvalidOperationException)
{
throwsException = true;
}
Assert.True(throwsException);
}
protected InMemoryRavenConfiguration CreateConfiguration(
string tenantId,
FileSystemDocument document,
string folderPropName,
InMemoryRavenConfiguration parentConfiguration)
{
var config = new InMemoryRavenConfiguration
{
Settings = new NameValueCollection(parentConfiguration.Settings),
};
SetupTenantConfiguration(config);
config.CustomizeValuesForFileSystemTenant(tenantId);
config.Settings[Constants.FileSystem.Storage] = parentConfiguration.FileSystem.DefaultStorageTypeName;
foreach (var setting in document.Settings)
{
config.Settings[setting.Key] = setting.Value;
}
Unprotect(document);
foreach (var securedSetting in document.SecuredSettings)
{
config.Settings[securedSetting.Key] = securedSetting.Value;
}
config.Settings[folderPropName] = config.Settings[folderPropName].ToFullPath(parentConfiguration.FileSystem.DataDirectory);
config.FileSystemName = tenantId;
config.Initialize();
config.CopyParentSettings(parentConfiguration);
return(config);
}
public void Unprotect(FileSystemDocument configDocument)
{
if (configDocument.SecuredSettings == null)
{
configDocument.SecuredSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
return;
}
foreach (var prop in configDocument.SecuredSettings.ToList())
{
if (prop.Value == null)
{
continue;
}
var bytes = Convert.FromBase64String(prop.Value);
var entrophy = Encoding.UTF8.GetBytes(prop.Key);
try
{
var unprotectedValue = ProtectedData.Unprotect(bytes, entrophy, DataProtectionScope.CurrentUser);
configDocument.SecuredSettings[prop.Key] = Encoding.UTF8.GetString(unprotectedValue);
}
catch (Exception e)
{
Logger.WarnException("Could not unprotect secured db data " + prop.Key + " setting the value to '<data could not be decrypted>'", e);
configDocument.SecuredSettings[prop.Key] = Constants.DataCouldNotBeDecrypted;
}
}
}
protected BaseBackupOperation(RavenFileSystem filesystem, string backupSourceDirectory, string backupDestinationDirectory, bool incrementalBackup, FileSystemDocument filesystemDocument)
{
if (filesystem == null)
{
throw new ArgumentNullException("filesystem");
}
if (filesystemDocument == null)
{
throw new ArgumentNullException("filesystemDocument");
}
if (backupSourceDirectory == null)
{
throw new ArgumentNullException("backupSourceDirectory");
}
if (backupDestinationDirectory == null)
{
throw new ArgumentNullException("backupDestinationDirectory");
}
this.filesystem = filesystem;
this.backupSourceDirectory = backupSourceDirectory.ToFullPath();
this.backupDestinationDirectory = backupDestinationDirectory.ToFullPath();
this.incrementalBackup = incrementalBackup;
this.filesystemDocument = filesystemDocument;
}
public BackupOperation(RavenFileSystem filesystem, string backupSourceDirectory,
string backupDestinationDirectory, StorageEnvironment env, bool incrementalBackup,
FileSystemDocument fileSystemDocument)
: base(filesystem, backupSourceDirectory, backupDestinationDirectory, incrementalBackup, fileSystemDocument)
{
if (env == null)
{
throw new ArgumentNullException("env");
}
this.env = env;
}
public FileSystemDocument ShowFileSystemDocument(Project project)
{
ProjectDocumtenKey key = new FileSystemDocumentKey(project);
FileSystemDocument document = GetDocument(key) as FileSystemDocument;
if (document == null)
{
document = new FileSystemDocument(project);
}
document.Show(this);
return(document);
}
public Task StartBackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupDestinationDirectory, bool incrementalBackup,
FileSystemDocument fileSystemDocument, ResourceBackupState state, CancellationToken token)
{
if (new InstanceParameters(instance).Recovery == false)
{
throw new InvalidOperationException("Cannot start backup operation since the recovery option is disabled. In order to enable the recovery please set the RunInUnreliableYetFastModeThatIsNotSuitableForProduction configuration parameter value to false.");
}
var backupOperation = new BackupOperation(filesystem, systemDatabase.Configuration.DataDirectory, backupDestinationDirectory, incrementalBackup,
fileSystemDocument, state, token);
return(Task.Factory.StartNew(backupOperation.Execute));
}
public void StartBackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupDestinationDirectory, bool incrementalBackup,
FileSystemDocument fileSystemDocument)
{
if (tableStorage == null)
{
throw new InvalidOperationException("Cannot begin database backup - table store is not initialized");
}
var backupOperation = new BackupOperation(filesystem, systemDatabase.Configuration.DataDirectory,
backupDestinationDirectory, tableStorage.Environment, incrementalBackup, fileSystemDocument);
Task.Factory.StartNew(() =>
{
using (backupOperation)
backupOperation.Execute();
});
}
private static void MoveFile(FileSystemDocument fileSystemDocument, string boxPath, string subfolder = "")
{
var destination = Path.Combine(boxPath, subfolder);
EnsureDirectoryExists(destination);
try
{
var destinationFileName = Path.Combine(destination, fileSystemDocument.FilePath.Name);
Console.WriteLine($"Moving {fileSystemDocument} to {destinationFileName}");
File.Move(fileSystemDocument.FilePath.FullName, destinationFileName);
}
catch (IOException ex)
{
Console.WriteLine($"Could not move file {fileSystemDocument.FilePath.FullName}: {ex.Message}");
}
}
public Task StartBackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupDestinationDirectory, bool incrementalBackup,
FileSystemDocument fileSystemDocument, ResourceBackupState state, CancellationToken token)
{
if (tableStorage == null)
{
throw new InvalidOperationException("Cannot begin database backup - table store is not initialized");
}
var backupOperation = new BackupOperation(filesystem, systemDatabase.Configuration.DataDirectory,
backupDestinationDirectory, tableStorage.Environment, incrementalBackup, fileSystemDocument, state, token);
return(Task.Factory.StartNew(() =>
{
using (backupOperation)
{
backupOperation.Execute();
}
}));
}
public void Protect(FileSystemDocument configDocument)
{
if (configDocument.SecuredSettings == null)
{
configDocument.SecuredSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
return;
}
foreach (var prop in configDocument.SecuredSettings.ToList())
{
if (prop.Value == null)
{
continue;
}
var bytes = Encoding.UTF8.GetBytes(prop.Value);
var entrophy = Encoding.UTF8.GetBytes(prop.Key);
var protectedValue = ProtectedData.Protect(bytes, entrophy, DataProtectionScope.CurrentUser);
configDocument.SecuredSettings[prop.Key] = Convert.ToBase64String(protectedValue);
}
}
public async Task CreateFileSystemWhenExistingWillFail()
{
var client = NewAsyncClient();
var adminClient = client.Admin;
const string newFileSystemName = "testName_CreateFileSystemWhenExistingWillFail";
var fileSystemSpec = new FileSystemDocument
{
Id = Constants.FileSystem.Prefix + newFileSystemName,
Settings =
{
{ Constants.FileSystem.DataDirectory, Path.Combine("~", Path.Combine("FileSystems", newFileSystemName)) }
}
};
await adminClient.CreateFileSystemAsync(fileSystemSpec);
var names = await adminClient.GetNamesAsync();
Assert.Contains(newFileSystemName, names);
Assert.Throws <InvalidOperationException>(() => AsyncHelpers.RunSync(() => adminClient.CreateFileSystemAsync(fileSystemSpec)));
}
public async Task <HttpResponseMessage> Restore()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest));
}
var restoreStatus = new RestoreStatus {
Messages = new List <string>()
};
var restoreRequest = await ReadJsonObjectAsync <FilesystemRestoreRequest>();
FileSystemDocument filesystemDocument = null;
var fileSystemDocumentPath = FindFilesystemDocument(restoreRequest.BackupLocation);
if (!File.Exists(fileSystemDocumentPath))
{
throw new InvalidOperationException("Cannot restore when the Filesystem.Document file is missing in the backup folder: " + restoreRequest.BackupLocation);
}
var filesystemDocumentText = File.ReadAllText(fileSystemDocumentPath);
filesystemDocument = RavenJObject.Parse(filesystemDocumentText).JsonDeserialization <FileSystemDocument>();
var filesystemName = !string.IsNullOrWhiteSpace(restoreRequest.FilesystemName) ? restoreRequest.FilesystemName
: filesystemDocument == null ? null : filesystemDocument.Id;
if (string.IsNullOrWhiteSpace(filesystemName))
{
var errorMessage = (filesystemDocument == null || String.IsNullOrWhiteSpace(filesystemDocument.Id))
? BackupMethods.FilesystemDocumentFilename + " file is invalid - filesystem name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen."
: "A filesystem name must be supplied if the restore location does not contain a valid " + BackupMethods.FilesystemDocumentFilename + " file";
restoreStatus.Messages.Add(errorMessage);
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null);
return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest));
}
var ravenConfiguration = new RavenConfiguration
{
DatabaseName = filesystemName,
IsTenantDatabase = true
};
if (filesystemDocument != null)
{
foreach (var setting in filesystemDocument.Settings)
{
ravenConfiguration.Settings[setting.Key] = setting.Value;
}
}
if (File.Exists(Path.Combine(restoreRequest.BackupLocation, BackupMethods.Filename)))
{
ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.VoronTypeName;
}
else if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new")))
{
ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.EsentTypeName;
}
ravenConfiguration.CustomizeValuesForTenant(filesystemName);
ravenConfiguration.Initialize();
string documentDataDir;
ravenConfiguration.FileSystem.DataDirectory = ResolveTenantDataDirectory(restoreRequest.FilesystemLocation, filesystemName, out documentDataDir);
restoreRequest.FilesystemLocation = ravenConfiguration.FileSystem.DataDirectory;
DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, null);
bool defrag;
if (bool.TryParse(GetQueryStringValue("defrag"), out defrag))
{
restoreRequest.Defrag = defrag;
}
//TODO: add task to pending task list like in ImportDatabase
Task.Factory.StartNew(() =>
{
if (!string.IsNullOrWhiteSpace(restoreRequest.FilesystemLocation))
{
ravenConfiguration.FileSystem.DataDirectory = restoreRequest.FilesystemLocation;
}
using (var transactionalStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration))
{
transactionalStorage.Restore(restoreRequest, msg =>
{
restoreStatus.Messages.Add(msg);
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null,
RavenJObject.FromObject(restoreStatus), new RavenJObject(), null);
});
}
if (filesystemDocument == null)
{
return;
}
filesystemDocument.Settings["Raven/FileSystem/DataDir"] = documentDataDir;
if (restoreRequest.IndexesLocation != null)
{
filesystemDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation;
}
if (restoreRequest.JournalsLocation != null)
{
filesystemDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation;
}
filesystemDocument.Id = filesystemName;
DatabasesLandlord.SystemDatabase.Documents.Put("Raven/FileSystems/" + filesystemName, null, RavenJObject.FromObject(filesystemDocument),
new RavenJObject(), null);
restoreStatus.Messages.Add("The new filesystem was created");
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null,
RavenJObject.FromObject(restoreStatus), new RavenJObject(), null);
}, TaskCreationOptions.LongRunning);
return(GetEmptyMessage());
}
public void ShowProjectError(TMFile file)
{
FileSystemDocument document = this.ShowFileSystemDocument(_project);
document.ShowError(file);
}
public async Task CanRestoreBackupOfEncryptedFileSystem(string requestedStorage)
{
dataPath = NewDataPath("CanRestoreBackupOfEncryptedFileSystem", false);
using (var server = CreateServer(Ports[0], requestedStorage: requestedStorage, runInMemory: false, dataDirectory: dataPath))
{
var store = server.FilesStore;
var fs1Doc = new FileSystemDocument()
{
Id = "FS1",
Settings =
{
{ Constants.FileSystem.DataDirectory, Path.Combine(server.Configuration.FileSystem.DataDirectory, "FS1") },
{ Constants.ActiveBundles, "Encryption" }
},
SecuredSettings = new Dictionary <string, string>
{
{
"Raven/Encryption/Key", "arHd5ENxwieUCAGkf4Rns8oPWx3f6npDgAowtIAPox0="
},
{
"Raven/Encryption/Algorithm", "System.Security.Cryptography.DESCryptoServiceProvider, mscorlib"
},
},
};
await store.AsyncFilesCommands.Admin.CreateFileSystemAsync(fs1Doc, "FS1");
using (var session = store.OpenAsyncSession("FS1"))
{
session.RegisterUpload("test1.txt", StringToStream("Secret password"));
session.RegisterUpload("test2.txt", StringToStream("Security guard"));
await session.SaveChangesAsync();
}
await store.AsyncFilesCommands.ForFileSystem("FS1").Admin.StartBackup(backupDir, null, false, "FS1");
WaitForBackup(store.AsyncFilesCommands.ForFileSystem("FS1"), true);
string filesystemDir = Path.Combine(server.Configuration.FileSystem.DataDirectory, "FS2");
await store.AsyncFilesCommands.Admin.StartRestore(new FilesystemRestoreRequest
{
BackupLocation = backupDir,
FilesystemName = "FS2",
FilesystemLocation = filesystemDir
});
SpinWait.SpinUntil(() => store.AsyncFilesCommands.Admin.GetNamesAsync().Result.Contains("FS2"),
Debugger.IsAttached ? TimeSpan.FromMinutes(10) : TimeSpan.FromMinutes(1));
using (var session = server.DocumentStore.OpenAsyncSession(Constants.SystemDatabase))
{
var fs2Doc = await session.LoadAsync <FileSystemDocument>(Constants.FileSystem.Prefix + "FS2");
Assert.NotEqual(fs1Doc.SecuredSettings["Raven/Encryption/Key"], fs2Doc.SecuredSettings["Raven/Encryption/Key"]);
Assert.NotEqual(fs1Doc.SecuredSettings["Raven/Encryption/Algorithm"], fs2Doc.SecuredSettings["Raven/Encryption/Algorithm"]);
}
using (var session = store.OpenAsyncSession("FS2"))
{
var test1 = StreamToString(await session.DownloadAsync("test1.txt"));
Assert.Equal("Secret password", test1);
var test2 = StreamToString(await session.DownloadAsync("test2.txt"));
Assert.Equal("Security guard", test2);
}
}
Close();
EncryptionTestUtil.AssertPlainTextIsNotSavedInAnyFileInPath(new[]
{
"Secret password", "Security guard"
}, dataPath, s => true);
}
public async Task <HttpResponseMessage> Restore()
{
if (EnsureSystemDatabase() == false)
{
return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest));
}
var restoreStatus = new RestoreStatus {
Messages = new List <string>()
};
var restoreRequest = await ReadJsonObjectAsync <FilesystemRestoreRequest>();
var fileSystemDocumentPath = FindFilesystemDocument(restoreRequest.BackupLocation);
if (!File.Exists(fileSystemDocumentPath))
{
throw new InvalidOperationException("Cannot restore when the Filesystem.Document file is missing in the backup folder: " + restoreRequest.BackupLocation);
}
var filesystemDocumentText = File.ReadAllText(fileSystemDocumentPath);
FileSystemDocument filesystemDocument = RavenJObject.Parse(filesystemDocumentText).JsonDeserialization <FileSystemDocument>();
var filesystemName = !string.IsNullOrWhiteSpace(restoreRequest.FilesystemName) ? restoreRequest.FilesystemName
: filesystemDocument == null ? null : filesystemDocument.Id;
if (string.IsNullOrWhiteSpace(filesystemName))
{
var errorMessage = (filesystemDocument == null || String.IsNullOrWhiteSpace(filesystemDocument.Id))
? Constants.FilesystemDocumentFilename + " file is invalid - filesystem name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen."
: "A filesystem name must be supplied if the restore location does not contain a valid " + Constants.FilesystemDocumentFilename + " file";
restoreStatus.Messages.Add(errorMessage);
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null);
return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest));
}
var ravenConfiguration = new RavenConfiguration
{
FileSystemName = filesystemName,
};
if (filesystemDocument != null)
{
foreach (var setting in filesystemDocument.Settings)
{
ravenConfiguration.Settings[setting.Key] = setting.Value;
}
}
if (File.Exists(Path.Combine(restoreRequest.BackupLocation, BackupMethods.Filename)))
{
ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.VoronTypeName;
}
else if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new")))
{
ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.EsentTypeName;
}
ravenConfiguration.CustomizeValuesForFileSystemTenant(filesystemName);
ravenConfiguration.Initialize();
string documentDataDir;
ravenConfiguration.FileSystem.DataDirectory = ResolveTenantDataDirectory(restoreRequest.FilesystemLocation, filesystemName, out documentDataDir);
restoreRequest.FilesystemLocation = ravenConfiguration.FileSystem.DataDirectory;
string anotherRestoreResourceName;
if (IsAnotherRestoreInProgress(out anotherRestoreResourceName))
{
if (restoreRequest.RestoreStartTimeout.HasValue)
{
try
{
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter(TimeSpan.FromSeconds(restoreRequest.RestoreStartTimeout.Value));
var token = cts.Token;
do
{
await Task.Delay(500, token);
}while (IsAnotherRestoreInProgress(out anotherRestoreResourceName));
}
}
catch (OperationCanceledException)
{
return(GetMessageWithString(string.Format("Another restore is still in progress (resource name = {0}). Waited {1} seconds for other restore to complete.", anotherRestoreResourceName, restoreRequest.RestoreStartTimeout.Value), HttpStatusCode.ServiceUnavailable));
}
}
else
{
return(GetMessageWithString(string.Format("Another restore is in progress (resource name = {0})", anotherRestoreResourceName), HttpStatusCode.ServiceUnavailable));
}
}
Database.Documents.Put(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, RavenJObject.FromObject(new RestoreInProgress
{
Resource = filesystemName
}), new RavenJObject(), null);
DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, null);
bool defrag;
if (bool.TryParse(GetQueryStringValue("defrag"), out defrag))
{
restoreRequest.Defrag = defrag;
}
var task = Task.Factory.StartNew(() =>
{
if (!string.IsNullOrWhiteSpace(restoreRequest.FilesystemLocation))
{
ravenConfiguration.FileSystem.DataDirectory = restoreRequest.FilesystemLocation;
}
if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new")))
{
ravenConfiguration.DefaultStorageTypeName = "Esent";
}
else
{
ravenConfiguration.DefaultStorageTypeName = "Voron";
}
using (var transactionalStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration))
{
transactionalStorage.Restore(restoreRequest, msg =>
{
restoreStatus.Messages.Add(msg);
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null,
RavenJObject.FromObject(restoreStatus), new RavenJObject(), null);
});
}
if (filesystemDocument == null)
{
return;
}
filesystemDocument.Settings["Raven/FileSystem/DataDir"] = documentDataDir;
if (restoreRequest.IndexesLocation != null)
{
filesystemDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation;
}
if (restoreRequest.JournalsLocation != null)
{
filesystemDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation;
}
filesystemDocument.Id = filesystemName;
DatabasesLandlord.SystemDatabase.Documents.Put("Raven/FileSystems/" + filesystemName, null, RavenJObject.FromObject(filesystemDocument), new RavenJObject(), null);
restoreStatus.Messages.Add("The new filesystem was created");
DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null);
Database.Documents.Delete(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, null);
}, TaskCreationOptions.LongRunning);
long id;
Database.Tasks.AddTask(task, new TaskBasedOperationState(task), new TaskActions.PendingTaskDescription
{
StartTime = SystemTime.UtcNow,
TaskType = TaskActions.PendingTaskType.RestoreFilesystem,
Payload = "Restoring filesystem " + filesystemName + " from " + restoreRequest.BackupLocation
}, out id);
return(GetMessageWithObject(new
{
OperationId = id
}));
}
