public void Full_backup_must_backup_journals_that_we_havent_synced_yet()
{
RequireFileBasedPager();
var r = new Random();
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator)
{
AfterGatherInformationAction = () =>
{
Env.FlushLogToDataFile();
}
})
{
var syncResult = operation.SyncDataFile();
}
var voronDataDir = new VoronPathSetting(DataDir);
BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup"));
BackupMethods.Full.Restore(voronDataDir.Combine("voron-test.backup"), voronDataDir.Combine("backup-test.data"));
var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "backup-test.data"));
options.MaxLogFileSize = Env.Options.MaxLogFileSize;
using (var env = new StorageEnvironment(options))
{
}
}
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
Action <IOperationProgress> onProgress, RestoreResult restoreResult)
{
Debug.Assert(onProgress != null);
RestoreSettings restoreSettings = null;
var fullBackupPath = GetBackupPath(backupPath);
using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
{
foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
{
var directory = zipEntries.Key;
if (string.IsNullOrWhiteSpace(directory))
{
foreach (var zipEntry in zipEntries)
{
if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
{
using (var entryStream = zipEntry.Open())
{
var stream = RestoreFromConfiguration.BackupEncryptionSettings?.EncryptionMode == EncryptionMode.UseDatabaseKey ?
new DecryptingXChaCha20Oly1305Stream(entryStream, Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey))
: entryStream;
var json = context.Read(stream, "read database settings for restore");
json.BlittableValidation();
restoreSettings = JsonDeserializationServer.RestoreSettings(json);
restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
{
throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
}
if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
{
throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
}
}
}
}
continue;
}
var voronDataDirectory = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
? voronDataDirectory
: voronDataDirectory.Combine(directory);
BackupMethods.Full.Restore(
zipEntries,
restoreDirectory,
journalDir: null,
onProgress: message =>
{
restoreResult.AddInfo(message);
restoreResult.SnapshotRestore.ReadCount++;
onProgress.Invoke(restoreResult.Progress);
},
cancellationToken: _operationCancelToken.Token);
}
}
if (restoreSettings == null)
{
throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
}
return(restoreSettings);
}
private RestoreSettings SnapshotRestore(
string backupPath,
string dataDirectory,
Action <IOperationProgress> onProgress,
RestoreResult restoreResult)
{
Debug.Assert(onProgress != null);
RestoreSettings restoreSettings = null;
var voronBackupPath = new VoronPathSetting(backupPath);
var voronDataDirectory = new VoronPathSetting(dataDirectory);
using (var zip = ZipFile.Open(voronBackupPath.FullPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
{
foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
{
var directory = zipEntries.Key;
if (string.IsNullOrWhiteSpace(directory))
{
foreach (var zipEntry in zipEntries)
{
if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
{
using (var entryStream = zipEntry.Open())
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var json = context.Read(entryStream, "read database settings for restore");
json.BlittableValidation();
restoreSettings = JsonDeserializationServer.RestoreSettings(json);
restoreSettings.DatabaseRecord.DatabaseName = _restoreConfiguration.DatabaseName;
DatabaseHelper.Validate(_restoreConfiguration.DatabaseName, restoreSettings.DatabaseRecord);
if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
{
throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
}
if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
{
throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
}
}
}
}
continue;
}
var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
? voronDataDirectory
: voronDataDirectory.Combine(directory);
BackupMethods.Full.Restore(
zipEntries,
restoreDirectory,
journalDir: null,
onProgress: message =>
{
restoreResult.AddInfo(message);
restoreResult.SnapshotRestore.ReadCount++;
onProgress.Invoke(restoreResult.Progress);
},
cancellationToken: _operationCancelToken.Token);
}
}
if (restoreSettings == null)
{
throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
}
return(restoreSettings);
}
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
Action <IOperationProgress> onProgress, RestoreResult restoreResult)
{
Debug.Assert(onProgress != null);
RestoreSettings restoreSettings = null;
var fullBackupPath = GetBackupPath(backupPath);
using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
{
var restorePath = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
if (Directory.Exists(restorePath.FullPath) == false)
{
Directory.CreateDirectory(restorePath.FullPath);
}
// validate free space
var snapshotSize = zip.Entries.Sum(entry => entry.Length);
BackupHelper.AssertFreeSpaceForSnapshot(restorePath.FullPath, snapshotSize, "restore a backup", Logger);
foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
{
var directory = zipEntries.Key;
if (string.IsNullOrWhiteSpace(directory))
{
foreach (var zipEntry in zipEntries)
{
if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
{
await using (var entryStream = zipEntry.Open())
{
var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null
? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)
: null;
await using (var stream = GetInputStream(entryStream, snapshotEncryptionKey))
{
var json = await context.ReadForMemoryAsync(stream, "read database settings for restore");
json.BlittableValidation();
restoreSettings = JsonDeserializationServer.RestoreSettings(json);
restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
{
throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
}
if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
{
throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
}
}
}
}
}
continue;
}
var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
? restorePath
: restorePath.Combine(directory);
var isSubDirectory = PathUtil.IsSubDirectory(restoreDirectory.FullPath, restorePath.FullPath);
if (isSubDirectory == false)
{
var extensions = zipEntries
.Select(x => Path.GetExtension(x.Name))
.Distinct()
.ToArray();
if (extensions.Length != 1 || string.Equals(extensions[0], TableValueCompressor.CompressionRecoveryExtension, StringComparison.OrdinalIgnoreCase) == false)
{
throw new InvalidOperationException($"Encountered invalid directory '{directory}' in snapshot file with following file extensions: {string.Join(", ", extensions)}");
}
// this enables backward compatibility of snapshot backups with compression recovery files before fix was made in RavenDB-17173
// the underlying issue was that we were putting full path when compression recovery files were backed up using snapshot
// because of that the end restore directory was not a sub-directory of a restore path
// which could result in a file already exists exception
// since restoring of compression recovery files is not mandatory then it is safe to skip them
continue;
}
BackupMethods.Full.Restore(
zipEntries,
restoreDirectory,
journalDir: null,
onProgress: message =>
{
restoreResult.AddInfo(message);
restoreResult.SnapshotRestore.ReadCount++;
onProgress.Invoke(restoreResult.Progress);
},
cancellationToken: _operationCancelToken.Token);
}
}
if (restoreSettings == null)
{
throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
}
return(restoreSettings);
}
