private void EnsureFileSystemHasRequiredSettings(string id, FileSystemDocument fsDoc) { if (!fsDoc.Settings.ContainsKey(Constants.FileSystem.DataDirectory)) { fsDoc.Settings[Constants.FileSystem.DataDirectory] = "~/FileSystems/" + id; } }
public async Task CreateFileSystemWhenExistingWillFail() { var client = NewAsyncClient(); var adminClient = client.Admin; const string newFileSystemName = "testName_CreateFileSystemWhenExistingWillFail"; var fileSystemSpec = new FileSystemDocument { Id = "Raven/FileSystem/" + newFileSystemName, Settings = { { "Raven/FileSystem/DataDir", Path.Combine("~", Path.Combine("FileSystems", newFileSystemName)) } } }; await adminClient.CreateFileSystemAsync(fileSystemSpec, newFileSystemName); var names = await adminClient.GetNamesAsync(); Assert.Contains(newFileSystemName, names); bool throwsException = false; try { await adminClient.CreateFileSystemAsync(fileSystemSpec, newFileSystemName); } catch (InvalidOperationException) { throwsException = true; } Assert.True(throwsException); }
protected InMemoryRavenConfiguration CreateConfiguration( string tenantId, FileSystemDocument document, string folderPropName, InMemoryRavenConfiguration parentConfiguration) { var config = new InMemoryRavenConfiguration { Settings = new NameValueCollection(parentConfiguration.Settings), }; SetupTenantConfiguration(config); config.CustomizeValuesForFileSystemTenant(tenantId); config.Settings[Constants.FileSystem.Storage] = parentConfiguration.FileSystem.DefaultStorageTypeName; foreach (var setting in document.Settings) { config.Settings[setting.Key] = setting.Value; } Unprotect(document); foreach (var securedSetting in document.SecuredSettings) { config.Settings[securedSetting.Key] = securedSetting.Value; } config.Settings[folderPropName] = config.Settings[folderPropName].ToFullPath(parentConfiguration.FileSystem.DataDirectory); config.FileSystemName = tenantId; config.Initialize(); config.CopyParentSettings(parentConfiguration); return(config); }
public void Unprotect(FileSystemDocument configDocument) { if (configDocument.SecuredSettings == null) { configDocument.SecuredSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); return; } foreach (var prop in configDocument.SecuredSettings.ToList()) { if (prop.Value == null) { continue; } var bytes = Convert.FromBase64String(prop.Value); var entrophy = Encoding.UTF8.GetBytes(prop.Key); try { var unprotectedValue = ProtectedData.Unprotect(bytes, entrophy, DataProtectionScope.CurrentUser); configDocument.SecuredSettings[prop.Key] = Encoding.UTF8.GetString(unprotectedValue); } catch (Exception e) { Logger.WarnException("Could not unprotect secured db data " + prop.Key + " setting the value to '<data could not be decrypted>'", e); configDocument.SecuredSettings[prop.Key] = Constants.DataCouldNotBeDecrypted; } } }
protected BaseBackupOperation(RavenFileSystem filesystem, string backupSourceDirectory, string backupDestinationDirectory, bool incrementalBackup, FileSystemDocument filesystemDocument) { if (filesystem == null) { throw new ArgumentNullException("filesystem"); } if (filesystemDocument == null) { throw new ArgumentNullException("filesystemDocument"); } if (backupSourceDirectory == null) { throw new ArgumentNullException("backupSourceDirectory"); } if (backupDestinationDirectory == null) { throw new ArgumentNullException("backupDestinationDirectory"); } this.filesystem = filesystem; this.backupSourceDirectory = backupSourceDirectory.ToFullPath(); this.backupDestinationDirectory = backupDestinationDirectory.ToFullPath(); this.incrementalBackup = incrementalBackup; this.filesystemDocument = filesystemDocument; }
public BackupOperation(RavenFileSystem filesystem, string backupSourceDirectory, string backupDestinationDirectory, StorageEnvironment env, bool incrementalBackup, FileSystemDocument fileSystemDocument) : base(filesystem, backupSourceDirectory, backupDestinationDirectory, incrementalBackup, fileSystemDocument) { if (env == null) { throw new ArgumentNullException("env"); } this.env = env; }
public FileSystemDocument ShowFileSystemDocument(Project project) { ProjectDocumtenKey key = new FileSystemDocumentKey(project); FileSystemDocument document = GetDocument(key) as FileSystemDocument; if (document == null) { document = new FileSystemDocument(project); } document.Show(this); return(document); }
public Task StartBackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupDestinationDirectory, bool incrementalBackup, FileSystemDocument fileSystemDocument, ResourceBackupState state, CancellationToken token) { if (new InstanceParameters(instance).Recovery == false) { throw new InvalidOperationException("Cannot start backup operation since the recovery option is disabled. In order to enable the recovery please set the RunInUnreliableYetFastModeThatIsNotSuitableForProduction configuration parameter value to false."); } var backupOperation = new BackupOperation(filesystem, systemDatabase.Configuration.DataDirectory, backupDestinationDirectory, incrementalBackup, fileSystemDocument, state, token); return(Task.Factory.StartNew(backupOperation.Execute)); }
public void StartBackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupDestinationDirectory, bool incrementalBackup, FileSystemDocument fileSystemDocument) { if (tableStorage == null) { throw new InvalidOperationException("Cannot begin database backup - table store is not initialized"); } var backupOperation = new BackupOperation(filesystem, systemDatabase.Configuration.DataDirectory, backupDestinationDirectory, tableStorage.Environment, incrementalBackup, fileSystemDocument); Task.Factory.StartNew(() => { using (backupOperation) backupOperation.Execute(); }); }
private static void MoveFile(FileSystemDocument fileSystemDocument, string boxPath, string subfolder = "") { var destination = Path.Combine(boxPath, subfolder); EnsureDirectoryExists(destination); try { var destinationFileName = Path.Combine(destination, fileSystemDocument.FilePath.Name); Console.WriteLine($"Moving {fileSystemDocument} to {destinationFileName}"); File.Move(fileSystemDocument.FilePath.FullName, destinationFileName); } catch (IOException ex) { Console.WriteLine($"Could not move file {fileSystemDocument.FilePath.FullName}: {ex.Message}"); } }
public Task StartBackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupDestinationDirectory, bool incrementalBackup, FileSystemDocument fileSystemDocument, ResourceBackupState state, CancellationToken token) { if (tableStorage == null) { throw new InvalidOperationException("Cannot begin database backup - table store is not initialized"); } var backupOperation = new BackupOperation(filesystem, systemDatabase.Configuration.DataDirectory, backupDestinationDirectory, tableStorage.Environment, incrementalBackup, fileSystemDocument, state, token); return(Task.Factory.StartNew(() => { using (backupOperation) { backupOperation.Execute(); } })); }
public void Protect(FileSystemDocument configDocument) { if (configDocument.SecuredSettings == null) { configDocument.SecuredSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); return; } foreach (var prop in configDocument.SecuredSettings.ToList()) { if (prop.Value == null) { continue; } var bytes = Encoding.UTF8.GetBytes(prop.Value); var entrophy = Encoding.UTF8.GetBytes(prop.Key); var protectedValue = ProtectedData.Protect(bytes, entrophy, DataProtectionScope.CurrentUser); configDocument.SecuredSettings[prop.Key] = Convert.ToBase64String(protectedValue); } }
public async Task CreateFileSystemWhenExistingWillFail() { var client = NewAsyncClient(); var adminClient = client.Admin; const string newFileSystemName = "testName_CreateFileSystemWhenExistingWillFail"; var fileSystemSpec = new FileSystemDocument { Id = Constants.FileSystem.Prefix + newFileSystemName, Settings = { { Constants.FileSystem.DataDirectory, Path.Combine("~", Path.Combine("FileSystems", newFileSystemName)) } } }; await adminClient.CreateFileSystemAsync(fileSystemSpec); var names = await adminClient.GetNamesAsync(); Assert.Contains(newFileSystemName, names); Assert.Throws <InvalidOperationException>(() => AsyncHelpers.RunSync(() => adminClient.CreateFileSystemAsync(fileSystemSpec))); }
public async Task <HttpResponseMessage> Restore() { if (EnsureSystemDatabase() == false) { return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest)); } var restoreStatus = new RestoreStatus { Messages = new List <string>() }; var restoreRequest = await ReadJsonObjectAsync <FilesystemRestoreRequest>(); FileSystemDocument filesystemDocument = null; var fileSystemDocumentPath = FindFilesystemDocument(restoreRequest.BackupLocation); if (!File.Exists(fileSystemDocumentPath)) { throw new InvalidOperationException("Cannot restore when the Filesystem.Document file is missing in the backup folder: " + restoreRequest.BackupLocation); } var filesystemDocumentText = File.ReadAllText(fileSystemDocumentPath); filesystemDocument = RavenJObject.Parse(filesystemDocumentText).JsonDeserialization <FileSystemDocument>(); var filesystemName = !string.IsNullOrWhiteSpace(restoreRequest.FilesystemName) ? restoreRequest.FilesystemName : filesystemDocument == null ? null : filesystemDocument.Id; if (string.IsNullOrWhiteSpace(filesystemName)) { var errorMessage = (filesystemDocument == null || String.IsNullOrWhiteSpace(filesystemDocument.Id)) ? BackupMethods.FilesystemDocumentFilename + " file is invalid - filesystem name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen." : "A filesystem name must be supplied if the restore location does not contain a valid " + BackupMethods.FilesystemDocumentFilename + " file"; restoreStatus.Messages.Add(errorMessage); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null); return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest)); } var ravenConfiguration = new RavenConfiguration { DatabaseName = filesystemName, IsTenantDatabase = true }; if (filesystemDocument != null) { foreach (var setting in filesystemDocument.Settings) { ravenConfiguration.Settings[setting.Key] = setting.Value; } } if (File.Exists(Path.Combine(restoreRequest.BackupLocation, BackupMethods.Filename))) { ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.VoronTypeName; } else if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new"))) { ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.EsentTypeName; } ravenConfiguration.CustomizeValuesForTenant(filesystemName); ravenConfiguration.Initialize(); string documentDataDir; ravenConfiguration.FileSystem.DataDirectory = ResolveTenantDataDirectory(restoreRequest.FilesystemLocation, filesystemName, out documentDataDir); restoreRequest.FilesystemLocation = ravenConfiguration.FileSystem.DataDirectory; DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, null); bool defrag; if (bool.TryParse(GetQueryStringValue("defrag"), out defrag)) { restoreRequest.Defrag = defrag; } //TODO: add task to pending task list like in ImportDatabase Task.Factory.StartNew(() => { if (!string.IsNullOrWhiteSpace(restoreRequest.FilesystemLocation)) { ravenConfiguration.FileSystem.DataDirectory = restoreRequest.FilesystemLocation; } using (var transactionalStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration)) { transactionalStorage.Restore(restoreRequest, msg => { restoreStatus.Messages.Add(msg); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); }); } if (filesystemDocument == null) { return; } filesystemDocument.Settings["Raven/FileSystem/DataDir"] = documentDataDir; if (restoreRequest.IndexesLocation != null) { filesystemDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation; } if (restoreRequest.JournalsLocation != null) { filesystemDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation; } filesystemDocument.Id = filesystemName; DatabasesLandlord.SystemDatabase.Documents.Put("Raven/FileSystems/" + filesystemName, null, RavenJObject.FromObject(filesystemDocument), new RavenJObject(), null); restoreStatus.Messages.Add("The new filesystem was created"); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); }, TaskCreationOptions.LongRunning); return(GetEmptyMessage()); }
public void ShowProjectError(TMFile file) { FileSystemDocument document = this.ShowFileSystemDocument(_project); document.ShowError(file); }
public async Task CanRestoreBackupOfEncryptedFileSystem(string requestedStorage) { dataPath = NewDataPath("CanRestoreBackupOfEncryptedFileSystem", false); using (var server = CreateServer(Ports[0], requestedStorage: requestedStorage, runInMemory: false, dataDirectory: dataPath)) { var store = server.FilesStore; var fs1Doc = new FileSystemDocument() { Id = "FS1", Settings = { { Constants.FileSystem.DataDirectory, Path.Combine(server.Configuration.FileSystem.DataDirectory, "FS1") }, { Constants.ActiveBundles, "Encryption" } }, SecuredSettings = new Dictionary <string, string> { { "Raven/Encryption/Key", "arHd5ENxwieUCAGkf4Rns8oPWx3f6npDgAowtIAPox0=" }, { "Raven/Encryption/Algorithm", "System.Security.Cryptography.DESCryptoServiceProvider, mscorlib" }, }, }; await store.AsyncFilesCommands.Admin.CreateFileSystemAsync(fs1Doc, "FS1"); using (var session = store.OpenAsyncSession("FS1")) { session.RegisterUpload("test1.txt", StringToStream("Secret password")); session.RegisterUpload("test2.txt", StringToStream("Security guard")); await session.SaveChangesAsync(); } await store.AsyncFilesCommands.ForFileSystem("FS1").Admin.StartBackup(backupDir, null, false, "FS1"); WaitForBackup(store.AsyncFilesCommands.ForFileSystem("FS1"), true); string filesystemDir = Path.Combine(server.Configuration.FileSystem.DataDirectory, "FS2"); await store.AsyncFilesCommands.Admin.StartRestore(new FilesystemRestoreRequest { BackupLocation = backupDir, FilesystemName = "FS2", FilesystemLocation = filesystemDir }); SpinWait.SpinUntil(() => store.AsyncFilesCommands.Admin.GetNamesAsync().Result.Contains("FS2"), Debugger.IsAttached ? TimeSpan.FromMinutes(10) : TimeSpan.FromMinutes(1)); using (var session = server.DocumentStore.OpenAsyncSession(Constants.SystemDatabase)) { var fs2Doc = await session.LoadAsync <FileSystemDocument>(Constants.FileSystem.Prefix + "FS2"); Assert.NotEqual(fs1Doc.SecuredSettings["Raven/Encryption/Key"], fs2Doc.SecuredSettings["Raven/Encryption/Key"]); Assert.NotEqual(fs1Doc.SecuredSettings["Raven/Encryption/Algorithm"], fs2Doc.SecuredSettings["Raven/Encryption/Algorithm"]); } using (var session = store.OpenAsyncSession("FS2")) { var test1 = StreamToString(await session.DownloadAsync("test1.txt")); Assert.Equal("Secret password", test1); var test2 = StreamToString(await session.DownloadAsync("test2.txt")); Assert.Equal("Security guard", test2); } } Close(); EncryptionTestUtil.AssertPlainTextIsNotSavedInAnyFileInPath(new[] { "Secret password", "Security guard" }, dataPath, s => true); }
public async Task <HttpResponseMessage> Restore() { if (EnsureSystemDatabase() == false) { return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest)); } var restoreStatus = new RestoreStatus { Messages = new List <string>() }; var restoreRequest = await ReadJsonObjectAsync <FilesystemRestoreRequest>(); var fileSystemDocumentPath = FindFilesystemDocument(restoreRequest.BackupLocation); if (!File.Exists(fileSystemDocumentPath)) { throw new InvalidOperationException("Cannot restore when the Filesystem.Document file is missing in the backup folder: " + restoreRequest.BackupLocation); } var filesystemDocumentText = File.ReadAllText(fileSystemDocumentPath); FileSystemDocument filesystemDocument = RavenJObject.Parse(filesystemDocumentText).JsonDeserialization <FileSystemDocument>(); var filesystemName = !string.IsNullOrWhiteSpace(restoreRequest.FilesystemName) ? restoreRequest.FilesystemName : filesystemDocument == null ? null : filesystemDocument.Id; if (string.IsNullOrWhiteSpace(filesystemName)) { var errorMessage = (filesystemDocument == null || String.IsNullOrWhiteSpace(filesystemDocument.Id)) ? Constants.FilesystemDocumentFilename + " file is invalid - filesystem name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen." : "A filesystem name must be supplied if the restore location does not contain a valid " + Constants.FilesystemDocumentFilename + " file"; restoreStatus.Messages.Add(errorMessage); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null); return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest)); } var ravenConfiguration = new RavenConfiguration { FileSystemName = filesystemName, }; if (filesystemDocument != null) { foreach (var setting in filesystemDocument.Settings) { ravenConfiguration.Settings[setting.Key] = setting.Value; } } if (File.Exists(Path.Combine(restoreRequest.BackupLocation, BackupMethods.Filename))) { ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.VoronTypeName; } else if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new"))) { ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.EsentTypeName; } ravenConfiguration.CustomizeValuesForFileSystemTenant(filesystemName); ravenConfiguration.Initialize(); string documentDataDir; ravenConfiguration.FileSystem.DataDirectory = ResolveTenantDataDirectory(restoreRequest.FilesystemLocation, filesystemName, out documentDataDir); restoreRequest.FilesystemLocation = ravenConfiguration.FileSystem.DataDirectory; string anotherRestoreResourceName; if (IsAnotherRestoreInProgress(out anotherRestoreResourceName)) { if (restoreRequest.RestoreStartTimeout.HasValue) { try { using (var cts = new CancellationTokenSource()) { cts.CancelAfter(TimeSpan.FromSeconds(restoreRequest.RestoreStartTimeout.Value)); var token = cts.Token; do { await Task.Delay(500, token); }while (IsAnotherRestoreInProgress(out anotherRestoreResourceName)); } } catch (OperationCanceledException) { return(GetMessageWithString(string.Format("Another restore is still in progress (resource name = {0}). Waited {1} seconds for other restore to complete.", anotherRestoreResourceName, restoreRequest.RestoreStartTimeout.Value), HttpStatusCode.ServiceUnavailable)); } } else { return(GetMessageWithString(string.Format("Another restore is in progress (resource name = {0})", anotherRestoreResourceName), HttpStatusCode.ServiceUnavailable)); } } Database.Documents.Put(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, RavenJObject.FromObject(new RestoreInProgress { Resource = filesystemName }), new RavenJObject(), null); DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, null); bool defrag; if (bool.TryParse(GetQueryStringValue("defrag"), out defrag)) { restoreRequest.Defrag = defrag; } var task = Task.Factory.StartNew(() => { if (!string.IsNullOrWhiteSpace(restoreRequest.FilesystemLocation)) { ravenConfiguration.FileSystem.DataDirectory = restoreRequest.FilesystemLocation; } if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new"))) { ravenConfiguration.DefaultStorageTypeName = "Esent"; } else { ravenConfiguration.DefaultStorageTypeName = "Voron"; } using (var transactionalStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration)) { transactionalStorage.Restore(restoreRequest, msg => { restoreStatus.Messages.Add(msg); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); }); } if (filesystemDocument == null) { return; } filesystemDocument.Settings["Raven/FileSystem/DataDir"] = documentDataDir; if (restoreRequest.IndexesLocation != null) { filesystemDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation; } if (restoreRequest.JournalsLocation != null) { filesystemDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation; } filesystemDocument.Id = filesystemName; DatabasesLandlord.SystemDatabase.Documents.Put("Raven/FileSystems/" + filesystemName, null, RavenJObject.FromObject(filesystemDocument), new RavenJObject(), null); restoreStatus.Messages.Add("The new filesystem was created"); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); Database.Documents.Delete(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, null); }, TaskCreationOptions.LongRunning); long id; Database.Tasks.AddTask(task, new TaskBasedOperationState(task), new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.RestoreFilesystem, Payload = "Restoring filesystem " + filesystemName + " from " + restoreRequest.BackupLocation }, out id); return(GetMessageWithObject(new { OperationId = id })); }