public void Full_backup_must_backup_journals_that_we_havent_synced_yet() { RequireFileBasedPager(); var r = new Random(); var bytes = new byte[512]; for (int i = 0; i < 10; i++) { using (var tx = Env.WriteTransaction()) { Tree tree = tx.CreateTree("tree"); for (int j = 0; j < 100; j++) { r.NextBytes(bytes); tree.Add(new string((char)j, 1000), bytes); } tx.Commit(); } } Env.FlushLogToDataFile(); for (int i = 0; i < 10; i++) { using (var tx = Env.WriteTransaction()) { Tree tree = tx.CreateTree("tree"); for (int j = 0; j < 100; j++) { r.NextBytes(bytes); tree.Add(new string((char)j, 1000), bytes); } tx.Commit(); } } using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator) { AfterGatherInformationAction = () => { Env.FlushLogToDataFile(); } }) { var syncResult = operation.SyncDataFile(); } var voronDataDir = new VoronPathSetting(DataDir); BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup")); BackupMethods.Full.Restore(voronDataDir.Combine("voron-test.backup"), voronDataDir.Combine("backup-test.data")); var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "backup-test.data")); options.MaxLogFileSize = Env.Options.MaxLogFileSize; using (var env = new StorageEnvironment(options)) { } }
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { using (var entryStream = zipEntry.Open()) { var stream = RestoreFromConfiguration.BackupEncryptionSettings?.EncryptionMode == EncryptionMode.UseDatabaseKey ? new DecryptingXChaCha20Oly1305Stream(entryStream, Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)) : entryStream; var json = context.Read(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } continue; } var voronDataDirectory = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? voronDataDirectory : voronDataDirectory.Combine(directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
private RestoreSettings SnapshotRestore( string backupPath, string dataDirectory, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var voronBackupPath = new VoronPathSetting(backupPath); var voronDataDirectory = new VoronPathSetting(dataDirectory); using (var zip = ZipFile.Open(voronBackupPath.FullPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8)) { foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { using (var entryStream = zipEntry.Open()) using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) { var json = context.Read(entryStream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = _restoreConfiguration.DatabaseName; DatabaseHelper.Validate(_restoreConfiguration.DatabaseName, restoreSettings.DatabaseRecord); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? voronDataDirectory : voronDataDirectory.Combine(directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); if (Directory.Exists(restorePath.FullPath) == false) { Directory.CreateDirectory(restorePath.FullPath); } // validate free space var snapshotSize = zip.Entries.Sum(entry => entry.Length); BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger); foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { await using (var entryStream = zipEntry.Open()) { var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey) : null; await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey)) { var json = await context.ReadForMemoryAsync(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } } continue; } var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? restorePath : restorePath.Combine(directory); var isSubDirectory = PathUtil.IsSubDirectory(restoreDirectory.FullPath, restorePath.FullPath); if (isSubDirectory == false) { var extensions = zipEntries .Select(x => Path.GetExtension(x.Name)) .Distinct() .ToArray(); if (extensions.Length != 1 || string.Equals(extensions[0], TableValueCompressor.CompressionRecoveryExtension, StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException($"Encountered invalid directory '{directory}' in snapshot file with following file extensions: {string.Join(", ", extensions)}"); } // this enables backward compatibility of snapshot backups with compression recovery files before fix was made in RavenDB-17173 // the underlying issue was that we were putting full path when compression recovery files were backed up using snapshot // because of that the end restore directory was not a sub-directory of a restore path // which could result in a file already exists exception // since restoring of compression recovery files is not mandatory then it is safe to skip them continue; } BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }