private RestoreSettings SnapshotRestore( string backupPath, string dataDirectory, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; BackupMethods.Full.Restore( backupPath, dataDirectory, journalDir: Path.Combine(dataDirectory, "Journal"), settingsKey: RestoreSettings.SettingsFileName, onSettings: settingsStream => { //TODO: decrypt this file using the _restoreConfiguration.EncryptionKey //http://issues.hibernatingrhinos.com/issue/RavenDB-7546 using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { var json = context.Read(settingsStream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = _restoreConfiguration.DatabaseName; DatabaseHelper.Validate(_restoreConfiguration.DatabaseName, restoreSettings.DatabaseRecord); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } },
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress) { try { if (onProgress == null) { onProgress = _ => { } } ; var result = new RestoreResult { DataDirectory = _restoreConfiguration.DataDirectory }; Stopwatch sw = null; RestoreSettings restoreSettings = null; var firstFile = _filesToRestore[0]; var extension = Path.GetExtension(firstFile); var snapshotRestore = false; if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension) { onProgress.Invoke(result.Progress); snapshotRestore = true; sw = Stopwatch.StartNew(); // restore the snapshot restoreSettings = SnapshotRestore(firstFile, _restoreConfiguration.DataDirectory, onProgress, result); // removing the snapshot from the list of files _filesToRestore.RemoveAt(0); } else { result.SnapshotRestore.Skipped = true; result.SnapshotRestore.Processed = true; onProgress.Invoke(result.Progress); } var databaseName = _restoreConfiguration.DatabaseName; if (restoreSettings == null) { restoreSettings = new RestoreSettings { DatabaseRecord = new DatabaseRecord(databaseName) { // we only have a smuggler restore // use the encryption key to encrypt the database Encrypted = _hasEncryptionKey } }; DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord); } var databaseRecord = restoreSettings.DatabaseRecord; if (databaseRecord.Settings == null) { databaseRecord.Settings = new Dictionary <string, string>(); } databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"; databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = _restoreConfiguration.DataDirectory; if (_hasEncryptionKey) { // save the encryption key so we'll be able to access the database _serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey, databaseName, overwrite: false); } using (var database = new DocumentDatabase(databaseName, new RavenConfiguration(databaseName, ResourceType.Database) { Core = { DataDirectory = new PathSetting(_restoreConfiguration.DataDirectory), RunInMemory = false } }, _serverStore)) { // smuggler needs an existing document database to operate var options = InitializeOptions.SkipLoadingDatabaseRecord; if (snapshotRestore) { options |= InitializeOptions.GenerateNewDatabaseId; } database.Initialize(options); if (snapshotRestore) { result.SnapshotRestore.Processed = true; var summary = database.GetDatabaseSummary(); result.Documents.ReadCount += summary.DocumentsCount; result.Documents.Attachments.ReadCount += summary.AttachmentsCount; result.RevisionDocuments.ReadCount += summary.RevisionsCount; result.Indexes.ReadCount += databaseRecord.GetIndexesCount(); result.Identities.ReadCount += restoreSettings.Identities.Count; result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} " + $"files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms"); onProgress.Invoke(result.Progress); } SmugglerRestore(_restoreConfiguration.BackupLocation, database, databaseRecord, restoreSettings, onProgress, result); } result.Documents.Processed = true; result.RevisionDocuments.Processed = true; result.Indexes.Processed = true; result.Identities.Processed = true; onProgress.Invoke(result.Progress); databaseRecord.Topology = new DatabaseTopology(); // restoring to the current node only databaseRecord.Topology.Members.Add(_nodeTag); await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true); var index = await WriteIdentitiesAsync(databaseName, restoreSettings.Identities); await _serverStore.Cluster.WaitForIndexNotification(index); return(result); } catch (OperationCanceledException) { // database shutdown throw; } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations("Failed to restore database", e); } var alert = AlertRaised.Create( "Failed to restore database", $"Could not restore database named {_restoreConfiguration.DatabaseName}", AlertType.RestoreError, NotificationSeverity.Error, details: new ExceptionDetails(e)); _serverStore.NotificationCenter.Add(alert); // delete any files that we already created during the restore IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory); throw; } finally { _operationCancelToken.Dispose(); } }
private void SmugglerRestore( string backupDirectory, DocumentDatabase database, DatabaseRecord databaseRecord, RestoreSettings restoreSettings, Action <IOperationProgress> onProgress, RestoreResult result) { Debug.Assert(onProgress != null); // the files are already ordered by name // take only the files that are relevant for smuggler restore _filesToRestore = _filesToRestore .Where(file => { var extension = Path.GetExtension(file); return (Constants.Documents.PeriodicBackup.IncrementalBackupExtension.Equals(extension, StringComparison.OrdinalIgnoreCase) || Constants.Documents.PeriodicBackup.FullBackupExtension.Equals(extension, StringComparison.OrdinalIgnoreCase)); }) .OrderBy(x => x) .ToList(); if (_filesToRestore.Count == 0) { return; } // we do have at least one smuggler backup databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>(); databaseRecord.Indexes = new Dictionary <string, IndexDefinition>(); restoreSettings.Identities = new Dictionary <string, long>(); // restore the smuggler backup var options = new DatabaseSmugglerOptionsServerSide(); var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options); var destination = new DatabaseDestination(database); using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { for (var i = 0; i < _filesToRestore.Count - 1; i++) { var filePath = Path.Combine(backupDirectory, _filesToRestore[i]); ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options); } options.OperateOnTypes = oldOperateOnTypes; var lastFilePath = Path.Combine(backupDirectory, _filesToRestore.Last()); ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options, onIndexAction: indexAndType => { switch (indexAndType.Type) { case IndexType.AutoMap: case IndexType.AutoMapReduce: var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition; databaseRecord.AutoIndexes[autoIndexDefinition.Name] = PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type); break; case IndexType.Map: case IndexType.MapReduce: var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition; databaseRecord.Indexes[indexDefinition.Name] = indexDefinition; break; case IndexType.None: case IndexType.Faulty: break; default: throw new ArgumentOutOfRangeException(); } }, onIdentityAction: identity => restoreSettings.Identities[identity.Prefix] = identity.Value); } }
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress) { try { if (onProgress == null) { onProgress = _ => { } } ; var restoreResult = new RestoreResult { DataDirectory = _restoreConfiguration.DataDirectory, JournalStoragePath = _restoreConfiguration.JournalsStoragePath }; RestoreSettings restoreSettings = null; var firstFile = _filesToRestore[0]; var extension = Path.GetExtension(firstFile); var snapshotRestore = false; if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension) { // restore the snapshot restoreSettings = SnapshotRestore(firstFile, _restoreConfiguration.DataDirectory, _restoreConfiguration.JournalsStoragePath, onProgress, restoreResult); snapshotRestore = true; // removing the snapshot from the list of files _filesToRestore.RemoveAt(0); } var databaseName = _restoreConfiguration.DatabaseName; if (restoreSettings == null) { restoreSettings = new RestoreSettings { DatabaseRecord = new DatabaseRecord(databaseName) { // we only have a smuggler restore // use the encryption key to encrypt the database Encrypted = _hasEncryptionKey }, Identities = new Dictionary <string, long>() }; DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord); } var databaseRecord = restoreSettings.DatabaseRecord; if (databaseRecord.Settings == null) { databaseRecord.Settings = new Dictionary <string, string>(); } databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"; databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = _restoreConfiguration.DataDirectory; databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Storage.JournalsStoragePath)] = _restoreConfiguration.JournalsStoragePath; if (_hasEncryptionKey) { // save the encryption key so we'll be able to access the database _serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey, databaseName, overwrite: false); } using (var database = new DocumentDatabase(databaseName, new RavenConfiguration(databaseName, ResourceType.Database) { Core = { DataDirectory = new PathSetting(_restoreConfiguration.DataDirectory), RunInMemory = false }, Storage = { JournalsStoragePath = new PathSetting(_restoreConfiguration.JournalsStoragePath) } }, _serverStore)) { // smuggler needs an existing document database to operate var options = InitializeOptions.SkipLoadingDatabaseRecord; if (snapshotRestore) { options |= InitializeOptions.GenerateNewDatabaseId; } database.Initialize(options); SmugglerRestore(_restoreConfiguration.BackupLocation, database, databaseRecord, identity => restoreSettings.Identities[identity.Prefix] = identity.Value, onProgress, restoreResult); } databaseRecord.Topology = new DatabaseTopology(); // restoring to the current node databaseRecord.Topology.Members.Add(_nodeTag); // TODO: disribute key in cluster? // TODO: _restoreConfiguration.ReplicationFactor ? // TODO: _restoreConfiguration.TopologyMembers ? await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true); var index = await WriteIdentitiesAsync(databaseName, restoreSettings.Identities); await _serverStore.Cluster.WaitForIndexNotification(index); return(restoreResult); } catch (OperationCanceledException) { // database shutdown throw; } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations("Failed to restore database", e); } var alert = AlertRaised.Create( "Failed to restore database", $"Could not restore database named {_restoreConfiguration.DatabaseName}", AlertType.RestoreError, NotificationSeverity.Error, details: new ExceptionDetails(e)); _serverStore.NotificationCenter.Add(alert); // delete any files that we already created during the restore IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory); throw; } }
private RestoreSettings SnapshotRestore( string backupPath, string dataDirectory, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; using (var zip = ZipFile.Open(backupPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8)) { foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { using (var entryStream = zipEntry.Open()) using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { var json = context.Read(entryStream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = _restoreConfiguration.DatabaseName; DatabaseHelper.Validate(_restoreConfiguration.DatabaseName, restoreSettings.DatabaseRecord); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? dataDirectory : Path.Combine(dataDirectory, directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, restoreDirectory, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress) { var databaseName = _restoreConfiguration.DatabaseName; var result = new RestoreResult { DataDirectory = _restoreConfiguration.DataDirectory }; try { if (onProgress == null) { onProgress = _ => { } } ; Stopwatch sw = null; RestoreSettings restoreSettings = null; var firstFile = _filesToRestore[0]; var lastFile = _filesToRestore.Last(); var extension = Path.GetExtension(firstFile); var snapshotRestore = false; if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension) { onProgress.Invoke(result.Progress); snapshotRestore = true; sw = Stopwatch.StartNew(); // restore the snapshot restoreSettings = SnapshotRestore(firstFile, _restoreConfiguration.DataDirectory, onProgress, result); // removing the snapshot from the list of files _filesToRestore.RemoveAt(0); } else { result.SnapshotRestore.Skipped = true; result.SnapshotRestore.Processed = true; onProgress.Invoke(result.Progress); } if (restoreSettings == null) { restoreSettings = new RestoreSettings { DatabaseRecord = new DatabaseRecord(databaseName) { // we only have a smuggler restore // use the encryption key to encrypt the database Encrypted = _hasEncryptionKey } }; DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord); } var databaseRecord = restoreSettings.DatabaseRecord; if (databaseRecord.Settings == null) { databaseRecord.Settings = new Dictionary <string, string>(); } databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"; databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = _restoreConfiguration.DataDirectory; if (_hasEncryptionKey) { // save the encryption key so we'll be able to access the database _serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey, databaseName, overwrite: false); } var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup { var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}"; if (Logger.IsInfoEnabled) { Logger.Info(msg); } }); using (var database = new DocumentDatabase(databaseName, new RavenConfiguration(databaseName, ResourceType.Database) { Core = { DataDirectory = new PathSetting(_restoreConfiguration.DataDirectory), RunInMemory = false } }, _serverStore, addToInitLog)) { // smuggler needs an existing document database to operate var options = InitializeOptions.SkipLoadingDatabaseRecord; if (snapshotRestore) { options |= InitializeOptions.GenerateNewDatabaseId; } database.Initialize(options); if (snapshotRestore) { result.SnapshotRestore.Processed = true; var summary = database.GetDatabaseSummary(); result.Documents.ReadCount += summary.DocumentsCount; result.Documents.Attachments.ReadCount += summary.AttachmentsCount; result.RevisionDocuments.ReadCount += summary.RevisionsCount; result.Conflicts.ReadCount += summary.ConflictsCount; result.Indexes.ReadCount += databaseRecord.GetIndexesCount(); result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} " + $"files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms"); onProgress.Invoke(result.Progress); } using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { SmugglerRestore(_restoreConfiguration.BackupLocation, database, context, databaseRecord, onProgress, result); result.DatabaseRecord.Processed = true; result.Documents.Processed = true; result.RevisionDocuments.Processed = true; result.Conflicts.Processed = true; result.Indexes.Processed = true; onProgress.Invoke(result.Progress); databaseRecord.Topology = new DatabaseTopology(); // restoring to the current node only databaseRecord.Topology.Members.Add(_nodeTag); databaseRecord.Disabled = true; // we are currently restoring, shouldn't try to access it _serverStore.EnsureNotPassive(); var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true); await _serverStore.Cluster.WaitForIndexNotification(index); // restore identities & cmpXchg values RestoreFromLastFile(onProgress, database, lastFile, context, result); } } // after the db for restore is done, we can safely set the db status to active databaseRecord = _serverStore.LoadDatabaseRecord(databaseName, out _); databaseRecord.Disabled = false; var(updateIndex, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null); await _serverStore.Cluster.WaitForIndexNotification(updateIndex); return(result); } catch (Exception e) { if (Logger.IsOperationsEnabled) { Logger.Operations("Failed to restore database", e); } var alert = AlertRaised.Create( _restoreConfiguration.DatabaseName, "Failed to restore database", $"Could not restore database named {_restoreConfiguration.DatabaseName}", AlertType.RestoreError, NotificationSeverity.Error, details: new ExceptionDetails(e)); _serverStore.NotificationCenter.Add(alert); if (_serverStore.LoadDatabaseRecord(_restoreConfiguration.DatabaseName, out var _) == null) { // delete any files that we already created during the restore IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory); } else { var deleteResult = await _serverStore.DeleteDatabaseAsync(_restoreConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag }); await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index); } result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e.Message}"); onProgress.Invoke(result.Progress); throw; } finally { _operationCancelToken.Dispose(); } }