public async Task GetImport()
{
if (HttpContext.Request.Query.ContainsKey("file") == false &&
HttpContext.Request.Query.ContainsKey("url") == false)
{
throw new ArgumentException("'file' or 'url' are mandatory when using GET /smuggler/import");
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext);
using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var token = CreateOperationToken())
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token);
var result = smuggler.Execute();
WriteImportResult(context, result, ResponseBodyStream());
}
}
}
private async Task MigrateDatabase(long operationId, ImportInfo importInfo)
{
var startDocumentEtag = importInfo?.LastEtag ?? 0;
var url = $"{ServerUrl}/databases/{DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}";
var json = JsonConvert.SerializeObject(new DatabaseSmugglerOptionsServerSide());
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide();
var smuggler = new Documents.DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token);
smuggler.Execute();
}
}
public async Task Documents()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var stream = new ArrayStream(RequestBodyStream(), "Docs"))
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide
{
ReadLegacyEtag = true,
OperateOnTypes = DatabaseItemType.Documents
};
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options);
var result = smuggler.Execute();
var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId);
replicationSource.LastDocumentEtag = result.LegacyLastDocumentEtag;
replicationSource.Source = GetFromServer();
replicationSource.LastBatchSize = result.Documents.ReadCount + result.Tombstones.ReadCount;
replicationSource.LastModified = DateTime.UtcNow;
await SaveSourceReplicationInformation(replicationSource, context, documentId);
}
}
private async Task MigrateIndexes()
{
var response = await RunWithAuthRetry(async() =>
{
var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/indexes";
var request = new HttpRequestMessage(HttpMethod.Get, url);
var responseMessage = await Parameters.HttpClient.SendAsync(request, Parameters.CancelToken.Token);
return(responseMessage);
});
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export indexes from server: {Options.ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (var indexesStream = new ArrayStream(responseStream, "Indexes")) // indexes endpoint returns an array
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(indexesStream, context, Parameters.Database))
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
RemoveAnalyzers = Options.RemoveAnalyzers,
};
var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token);
smuggler.Execute();
}
}
private void ApplyBackwardCompatibility(DatabaseSmugglerOptionsServerSide options)
{
if (options == null)
{
return;
}
if (RequestRouter.TryGetClientVersion(HttpContext, out var version) == false)
{
return;
}
if (version.Major != RavenVersionAttribute.Instance.MajorVersion)
{
return;
}
// only all 4.0 and 4.1 less or equal to 41006
if (version.Revision < 50 || version.Revision > 41006)
{
return;
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
options.OperateOnTypes |= DatabaseItemType.Attachments;
}
}
private async Task MigrateIndexes()
{
var url = $"{ServerUrl}/databases/{DatabaseName}/indexes";
var request = new HttpRequestMessage(HttpMethod.Get, url);
var response = await _client.SendAsync(request, CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export indexes from server: {ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
// indexes endpoint returns an array
using (var indexesStream = new IndexesStream(responseStream))
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(indexesStream, context, Database))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide();
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token);
smuggler.Execute();
}
}
private async Task MigrateDocuments(string lastEtag)
{
var url = $"{ServerUrl}/databases/{DatabaseName}/streams/docs?etag={lastEtag}";
var request = new HttpRequestMessage(HttpMethod.Get, url);
var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export documents from server: {ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(responseStream, context, Database))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide
{
ReadLegacyEtag = true
};
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token);
// since we will be migrating indexes as separate task don't ensureStepsProcessed at this point
smuggler.Execute(ensureStepsProcessed: false);
}
}
/// <summary>
/// Restore CompareExchange, Identities and Subscriptions from smuggler file when restoring snapshot.
/// </summary>
/// <param name="onProgress"></param>
/// <param name="database"></param>
/// <param name="smugglerFile"></param>
/// <param name="context"></param>
protected async Task RestoreFromSmugglerFile(Action <IOperationProgress> onProgress, DocumentDatabase database, string smugglerFile, DocumentsOperationContext context)
{
var destination = new DatabaseDestination(database);
var smugglerOptions = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = DatabaseItemType.CompareExchange | DatabaseItemType.Identities | DatabaseItemType.Subscriptions,
SkipRevisionCreation = true
};
var lastPath = GetSmugglerBackupPath(smugglerFile);
using (var zip = await GetZipArchiveForSnapshot(lastPath))
{
foreach (var entry in zip.Entries)
{
if (entry.Name == RestoreSettings.SmugglerValuesFileName)
{
using (var input = entry.Open())
using (var inputStream = GetSnapshotInputStream(input, database.Name))
using (var uncompressed = new GZipStream(inputStream, CompressionMode.Decompress))
{
var source = new StreamSource(uncompressed, context, database);
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, smugglerOptions, onProgress: onProgress, token: _operationCancelToken.Token);
smuggler.Execute(ensureStepsProcessed: true, isLastFile: true);
}
break;
}
}
}
}
private void CreateBackup(
DatabaseSmugglerOptionsServerSide options, string backupFilePath,
long?startDocumentEtag, DocumentsOperationContext context,
Action <IOperationProgress> onProgress)
{
// the last etag is already included in the last backup
startDocumentEtag = startDocumentEtag == null ? 0 : ++startDocumentEtag;
using (var file = File.Open(backupFilePath, FileMode.CreateNew))
{
var smugglerSource = new DatabaseSource(_database, startDocumentEtag.Value);
var smugglerDestination = new StreamDestination(file, context, smugglerSource);
var smuggler = new Smuggler.Documents.DatabaseSmuggler(_database,
smugglerSource,
smugglerDestination,
_database.Time,
options: options,
result: _backupResult,
onProgress: onProgress,
token: TaskCancelToken.Token);
smuggler.Execute();
file.Flush(flushToDisk: true);
}
}
public StreamDocumentActions(BlittableJsonTextWriter writer, DocumentsOperationContext context, DatabaseSource source, DatabaseSmugglerOptionsServerSide options, Func <LazyStringValue, bool> filterMetadataProperty, string propertyName)
: base(writer, propertyName)
{
_context = context;
_source = source;
_options = options;
_filterMetadataProperty = filterMetadataProperty;
}
public async Task PostExport()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var operationId = GetLongQueryString("operationId", false) ?? Database.Operations.GetNextOperationId();
var startDocumentEtag = GetLongQueryString("startEtag", false) ?? 0;
var stream = TryGetRequestFromStream("DownloadOptions") ?? RequestBodyStream();
DatabaseSmugglerOptionsServerSide options;
using (context.GetManagedBuffer(out var buffer))
{
var firstRead = await stream.ReadAsync(buffer.Buffer.Array, buffer.Buffer.Offset, buffer.Buffer.Count);
buffer.Used = 0;
buffer.Valid = firstRead;
if (firstRead != 0)
{
var blittableJson = await context.ParseToMemoryAsync(stream, "DownloadOptions", BlittableJsonDocumentBuilder.UsageMode.None, buffer);
options = JsonDeserializationServer.DatabaseSmugglerOptions(blittableJson);
}
else
{
// no content, we'll use defaults
options = new DatabaseSmugglerOptionsServerSide();
}
}
ApplyBackwardCompatibility(options);
var token = CreateOperationToken();
var fileName = options.FileName;
if (string.IsNullOrEmpty(fileName))
{
fileName = $"Dump of {context.DocumentDatabase.Name} {SystemTime.UtcNow.ToString("yyyy-MM-dd HH-mm", CultureInfo.InvariantCulture)}";
}
var contentDisposition = "attachment; filename=" + Uri.EscapeDataString(fileName) + ".ravendbdump";
HttpContext.Response.Headers["Content-Disposition"] = contentDisposition;
try
{
await Database.Operations.AddOperation(
Database,
"Export database: " + Database.Name,
Operations.OperationType.DatabaseExport,
onProgress => Task.Run(() => ExportDatabaseInternal(options, startDocumentEtag, onProgress, context, token), token.Token), operationId, token : token);
}
catch (Exception)
{
HttpContext.Abort();
}
}
}
private Stream GetInputStream(Stream fileStream, DatabaseSmugglerOptionsServerSide options)
{
if (options.EncryptionKey != null)
{
return(new DecryptingXChaCha20Oly1305Stream(fileStream, Convert.FromBase64String(options.EncryptionKey)));
}
return(fileStream);
}
private Stream GetOutputStream(Stream fileStream, DatabaseSmugglerOptionsServerSide options)
{
if (options.EncryptionKey == null)
{
return(fileStream);
}
var key = options?.EncryptionKey;
return(new EncryptingXChaCha20Poly1305Stream(fileStream,
Convert.FromBase64String(key)));
}
private async Task MigrateDatabase(long operationId, ImportInfo importInfo)
{
var startDocumentEtag = importInfo?.LastEtag ?? 0;
var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}";
var databaseSmugglerOptionsServerSide = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = Options.OperateOnTypes,
RemoveAnalyzers = Options.RemoveAnalyzers
};
if (importInfo != null)
{
databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.Tombstones;
databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.CompareExchangeTombstones;
}
var json = JsonConvert.SerializeObject(databaseSmugglerOptionsServerSide);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var response = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
await using (var responseStream = await response.Content.ReadAsStreamAsync())
await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, Parameters.Database))
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
TransformScript = Options.TransformScript,
OperateOnTypes = Options.OperateOnTypes,
OperateOnDatabaseRecordTypes = Options.OperateOnDatabaseRecordTypes
};
var smuggler = new Documents.DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token);
await smuggler.ExecuteAsync();
}
}
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result)
{
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader, _csvHelperConfig);
_result = result;
var disposable = new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
});
return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.DevBuildNumber)));
}
private IOperationResult ExportDatabaseInternal(
DatabaseSmugglerOptionsServerSide options,
long startDocumentEtag,
Action <IOperationProgress> onProgress,
DocumentsOperationContext context,
OperationCancelToken token)
{
using (token)
{
var source = new DatabaseSource(Database, startDocumentEtag);
var destination = new StreamDestination(ResponseBodyStream(), context, source);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token);
return(smuggler.Execute());
}
}
public IDisposable Initialize(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, out long buildVersion)
{
buildVersion = ServerVersion.DevBuildNumber;
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader, _csvHelperConfig);
_result = result;
return(new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
}));
}
private void HandleLegacyIndexes()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var stream = new ArrayStream(RequestBodyStream(), nameof(DatabaseItemType.Indexes)))
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Indexes
};
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options);
smuggler.Execute();
}
}
private InternalBackupResult CreateBackup(
DatabaseSmugglerOptionsServerSide options, string backupFilePath, long?startDocumentEtag, long?startRaftIndex)
{
// the last etag is already included in the last backup
var currentBackupResults = new InternalBackupResult();
startDocumentEtag = startDocumentEtag == null ? 0 : ++startDocumentEtag;
startRaftIndex = startRaftIndex == null ? 0 : ++startRaftIndex;
using (Stream fileStream = File.Open(backupFilePath, FileMode.CreateNew))
using (var outputStream = GetOutputStream(fileStream))
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var smugglerSource = new DatabaseSource(_database, startDocumentEtag.Value, startRaftIndex.Value, _logger);
var smugglerDestination = new StreamDestination(outputStream, context, smugglerSource);
var smuggler = new DatabaseSmuggler(_database,
smugglerSource,
smugglerDestination,
_database.Time,
options: options,
result: _backupResult,
onProgress: _onProgress,
token: TaskCancelToken.Token);
smuggler.ExecuteAsync().Wait();
switch (outputStream)
{
case EncryptingXChaCha20Poly1305Stream encryptedStream:
encryptedStream.Flush(flushToDisk: true);
break;
case FileStream file:
file.Flush(flushToDisk: true);
break;
default:
throw new InvalidOperationException($" {outputStream.GetType()} not supported");
}
currentBackupResults.LastDocumentEtag = smugglerSource.LastEtag;
currentBackupResults.LastDatabaseChangeVector = smugglerSource.LastDatabaseChangeVector;
currentBackupResults.LastRaftIndex = smugglerSource.LastRaftIndex;
return(currentBackupResults);
}
}
private async Task <SmugglerResult> MigrateDatabase(string json, bool readLegacyEtag)
{
var response = await RunWithAuthRetry(async() =>
{
var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/studio-tasks/exportDatabase";
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var responseMessage = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token);
return(responseMessage);
});
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
await using (var responseStream = await response.Content.ReadAsStreamAsync())
await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, Parameters.Database))
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
#pragma warning disable 618
ReadLegacyEtag = readLegacyEtag,
#pragma warning restore 618
RemoveAnalyzers = Options.RemoveAnalyzers,
TransformScript = Options.TransformScript,
OperateOnTypes = Options.OperateOnTypes
};
var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token);
return(await smuggler.ExecuteAsync());
}
}
public IDisposable Initialize(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion)
{
_gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true);
_writer = new BlittableJsonTextWriter(_context, _gzipStream);
_options = options;
_writer.WriteStartObject();
_writer.WritePropertyName("BuildVersion");
_writer.WriteInteger(buildVersion);
return(new DisposableAction(() =>
{
_writer.WriteEndObject();
_writer.Dispose();
_gzipStream.Dispose();
}));
}
private void RestoreFromLastFile(Action <IOperationProgress> onProgress, DocumentDatabase database, string lastFile, DocumentsOperationContext context, RestoreResult result)
{
var destination = new DatabaseDestination(database);
var smugglerOptions = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = DatabaseItemType.CompareExchange | DatabaseItemType.Identities,
SkipRevisionCreation = true,
KeepOriginalChangeVector = true
};
var lastPath = Path.Combine(_restoreConfiguration.BackupLocation, lastFile);
if (Path.GetExtension(lastPath) == Constants.Documents.PeriodicBackup.SnapshotExtension)
{
using (var zip = ZipFile.Open(lastPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
{
foreach (var entry in zip.Entries)
{
if (entry.Name == RestoreSettings.SmugglerValuesFileName)
{
using (var input = entry.Open())
using (var uncompressed = new GZipStream(input, CompressionMode.Decompress))
{
var source = new StreamSource(uncompressed, context, database);
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, smugglerOptions, onProgress: onProgress, token: _operationCancelToken.Token);
smuggler.Execute();
}
break;
}
}
}
}
else
{
ImportSingleBackupFile(database, onProgress, null, lastPath, context, destination, smugglerOptions);
}
result.Identities.Processed = true;
result.CompareExchange.Processed = true;
onProgress.Invoke(result.Progress);
}
private async Task <IOperationResult> ExportDatabaseInternalAsync(
DatabaseSmugglerOptionsServerSide options,
long startDocumentEtag,
long startRaftIndex,
Action <IOperationProgress> onProgress,
DocumentsOperationContext context,
OperationCancelToken token)
{
using (token)
{
var source = new DatabaseSource(Database, startDocumentEtag, startRaftIndex, Logger);
await using (var outputStream = GetOutputStream(ResponseBodyStream(), options))
{
var destination = new StreamDestination(outputStream, context, source);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, onProgress: onProgress, token: token.Token);
return(await smuggler.ExecuteAsync());
}
}
}
private void ImportSingleBackupFile(DocumentDatabase database,
Action <IOperationProgress> onProgress, RestoreResult restoreResult,
string filePath, DocumentsOperationContext context,
DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options,
Action <IndexDefinitionAndType> onIndexAction = null)
{
using (var fileStream = File.Open(filePath, FileMode.Open))
using (var stream = new GZipStream(new BufferedStream(fileStream, 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var source = new StreamSource(stream, context, database))
{
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, options, result: restoreResult, onProgress: onProgress, token: _operationCancelToken.Token)
{
OnIndexAction = onIndexAction,
};
smuggler.Execute();
}
}
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.EtagForBackup;
}
}
var disposable = new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
});
return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.Build)));
}
public DatabaseSmuggler(DocumentDatabase database, ISmugglerSource source, ISmugglerDestination destination, SystemTime time,
DatabaseSmugglerOptionsServerSide options = null, SmugglerResult result = null, Action <IOperationProgress> onProgress = null,
CancellationToken token = default)
{
_database = database;
_source = source;
_destination = destination;
_options = options ?? new DatabaseSmugglerOptionsServerSide();
_result = result;
_token = token;
if (string.IsNullOrWhiteSpace(_options.TransformScript) == false)
{
_patcher = new SmugglerPatcher(_options, database);
}
_time = time;
_onProgress = onProgress ?? (progress => { });
}
public IAsyncDisposable InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, long buildVersion)
{
_gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true);
_writer = new AsyncBlittableJsonTextWriter(_context, _gzipStream);
_options = options;
SetupMetadataFilterMethod(_context);
_writer.WriteStartObject();
_writer.WritePropertyName("BuildVersion");
_writer.WriteInteger(buildVersion);
return(new AsyncDisposableAction(async() =>
{
_writer.WriteEndObject();
await _writer.DisposeAsync();
await _gzipStream.DisposeAsync();
}));
}
public async Task PostImport()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext, context);
using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var token = CreateOperationToken())
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token);
var result = smuggler.Execute();
WriteImportResult(context, result, ResponseBodyStream());
}
}
}
private void ApplyBackwardCompatibility(DatabaseSmugglerOptionsServerSide options)
{
if (options == null)
{
return;
}
if (((options.OperateOnTypes & DatabaseItemType.DatabaseRecord) != 0) &&
(options.OperateOnDatabaseRecordTypes == DatabaseRecordItemType.None))
{
options.OperateOnDatabaseRecordTypes = DatabaseSmugglerOptions.DefaultOperateOnDatabaseRecordTypes;
}
if (RequestRouter.TryGetClientVersion(HttpContext, out var version) == false)
{
return;
}
if (version.Major != RavenVersionAttribute.Instance.MajorVersion)
{
return;
}
#pragma warning disable 618
if (version.Minor < 2 && options.OperateOnTypes.HasFlag(DatabaseItemType.Counters))
#pragma warning restore 618
{
options.OperateOnTypes |= DatabaseItemType.CounterGroups;
}
// only all 4.0 and 4.1 less or equal to 41006
if (version.Revision < 50 || version.Revision > 41006)
{
return;
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
options.OperateOnTypes |= DatabaseItemType.Attachments;
}
}
private async Task ImportSingleBackupFile(DocumentDatabase database,
Action <IOperationProgress> onProgress, RestoreResult restoreResult,
string filePath, DocumentsOperationContext context,
DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options, bool isLastFile,
Action <IndexDefinitionAndType> onIndexAction = null,
Action <DatabaseRecord> onDatabaseRecordAction = null)
{
using (var fileStream = await GetStream(filePath))
using (var inputStream = GetInputStream(fileStream, database.MasterKey))
using (var gzipStream = new GZipStream(inputStream, CompressionMode.Decompress))
using (var source = new StreamSource(gzipStream, context, database))
{
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, options, result: restoreResult, onProgress: onProgress, token: _operationCancelToken.Token)
{
OnIndexAction = onIndexAction,
OnDatabaseRecordAction = onDatabaseRecordAction
};
smuggler.Execute(ensureStepsProcessed: false, isLastFile);
}
}
