private async Task MigrateIndexes() { var response = await RunWithAuthRetry(async() => { var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/indexes"; var request = new HttpRequestMessage(HttpMethod.Get, url); var responseMessage = await Parameters.HttpClient.SendAsync(request, Parameters.CancelToken.Token); return(responseMessage); }); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export indexes from server: {Options.ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } await using (var responseStream = await response.Content.ReadAsStreamAsync()) await using (var indexesStream = new ArrayStream(responseStream, "Indexes")) // indexes endpoint returns an array using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(indexesStream, context, Parameters.Database)) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { RemoveAnalyzers = Options.RemoveAnalyzers, }; var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token); await smuggler.ExecuteAsync(); } }
public async Task Documents() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var stream = new ArrayStream(RequestBodyStream(), "Docs")) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { #pragma warning disable CS0618 // Type or member is obsolete ReadLegacyEtag = true, #pragma warning restore CS0618 // Type or member is obsolete OperateOnTypes = DatabaseItemType.Documents }; var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options); var result = await smuggler.ExecuteAsync(); var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId); replicationSource.LastDocumentEtag = result.LegacyLastDocumentEtag; replicationSource.Source = GetFromServer(); replicationSource.LastBatchSize = result.Documents.ReadCount + result.Tombstones.ReadCount; replicationSource.LastModified = DateTime.UtcNow; await SaveSourceReplicationInformation(replicationSource, context, documentId); } }
private async Task HandleLegacyIndexesAsync() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var stream = new ArrayStream(RequestBodyStream(), nameof(DatabaseItemType.Indexes))) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.Indexes }; var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options); await smuggler.ExecuteAsync(); } }
private InternalBackupResult CreateBackup( DatabaseSmugglerOptionsServerSide options, string backupFilePath, long?startDocumentEtag, long?startRaftIndex) { // the last etag is already included in the last backup var currentBackupResults = new InternalBackupResult(); startDocumentEtag = startDocumentEtag == null ? 0 : ++startDocumentEtag; startRaftIndex = startRaftIndex == null ? 0 : ++startRaftIndex; using (Stream fileStream = File.Open(backupFilePath, FileMode.CreateNew)) using (var outputStream = GetOutputStream(fileStream)) using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var smugglerSource = new DatabaseSource(_database, startDocumentEtag.Value, startRaftIndex.Value, _logger); var smugglerDestination = new StreamDestination(outputStream, context, smugglerSource); var smuggler = new DatabaseSmuggler(_database, smugglerSource, smugglerDestination, _database.Time, options: options, result: _backupResult, onProgress: _onProgress, token: TaskCancelToken.Token); smuggler.ExecuteAsync().Wait(); switch (outputStream) { case EncryptingXChaCha20Poly1305Stream encryptedStream: encryptedStream.Flush(flushToDisk: true); break; case FileStream file: file.Flush(flushToDisk: true); break; default: throw new InvalidOperationException($" {outputStream.GetType()} not supported"); } currentBackupResults.LastDocumentEtag = smugglerSource.LastEtag; currentBackupResults.LastDatabaseChangeVector = smugglerSource.LastDatabaseChangeVector; currentBackupResults.LastRaftIndex = smugglerSource.LastRaftIndex; return(currentBackupResults); } }
public async Task CanImportNorthwind(string file) { using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5))) await using (var inputStream = GetType().Assembly.GetManifestResourceStream(file)) await using (var stream = new GZipStream(inputStream, CompressionMode.Decompress)) { Assert.NotNull(stream); using (DocumentDatabase database = CreateDocumentDatabase()) using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(stream, context, database)) { var destination = new DatabaseDestination(database); var smuggler = new DatabaseSmuggler(database, source, destination, database.Time, new DatabaseSmugglerOptionsServerSide { TransformScript = "this['Test'] = 'NewValue';" }); var result = await smuggler.ExecuteAsync().WithCancellation(cts.Token); Assert.Equal(1059, result.Documents.ReadCount); Assert.Equal(0, result.Documents.SkippedCount); Assert.Equal(0, result.Documents.ErroredCount); Assert.Equal(4, result.Indexes.ReadCount); Assert.Equal(0, result.Indexes.ErroredCount); Assert.Equal(0, result.RevisionDocuments.ReadCount); Assert.Equal(0, result.RevisionDocuments.ErroredCount); using (context.OpenReadTransaction()) { var countOfDocuments = database.DocumentsStorage.GetNumberOfDocuments(context); var countOfIndexes = database.IndexStore.GetIndexes().Count(); Assert.Equal(1059, countOfDocuments); Assert.Equal(3, countOfIndexes);// there are 4 in ravendbdump, but Raven/DocumentsByEntityName is skipped var doc = database.DocumentsStorage.Get(context, "orders/1"); string test; Assert.True(doc.Data.TryGet("Test", out test)); Assert.Equal("NewValue", test); } } } }
private async Task <SmugglerResult> MigrateDatabase(string json, bool readLegacyEtag) { var response = await RunWithAuthRetry(async() => { var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/studio-tasks/exportDatabase"; var content = new StringContent(json, Encoding.UTF8, "application/json"); var request = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; var responseMessage = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token); return(responseMessage); }); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } await using (var responseStream = await response.Content.ReadAsStreamAsync()) await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(stream, context, Parameters.Database)) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { #pragma warning disable 618 ReadLegacyEtag = readLegacyEtag, #pragma warning restore 618 RemoveAnalyzers = Options.RemoveAnalyzers, TransformScript = Options.TransformScript, OperateOnTypes = Options.OperateOnTypes }; var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token); return(await smuggler.ExecuteAsync()); } }
private async Task MigrateDocuments(string lastEtag) { var response = await RunWithAuthRetry(async() => { var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/streams/docs?etag={lastEtag}"; var request = new HttpRequestMessage(HttpMethod.Get, url); var responseMessage = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token); return(responseMessage); }); if (response.IsSuccessStatusCode == false) { var responseString = await response.Content.ReadAsStringAsync(); throw new InvalidOperationException($"Failed to export documents from server: {Options.ServerUrl}, " + $"status code: {response.StatusCode}, " + $"error: {responseString}"); } await using (var responseStream = await response.Content.ReadAsStreamAsync()) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var source = new StreamSource(responseStream, context, Parameters.Database)) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { #pragma warning disable 618 ReadLegacyEtag = true, #pragma warning restore 618 TransformScript = Options.TransformScript, OperateOnTypes = Options.OperateOnTypes }; var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token); // since we will be migrating indexes as separate task don't ensureStepsProcessed at this point await smuggler.ExecuteAsync(ensureStepsProcessed : false); } }
public async Task PostCreateSampleData() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { using (context.OpenReadTransaction()) { foreach (var collection in Database.DocumentsStorage.GetCollections(context)) { if (collection.Count > 0) { throw new InvalidOperationException("You cannot create sample data in a database that already contains documents"); } } } var operateOnTypesAsString = GetStringValuesQueryString("operateOnTypes", required: false); var operateOnTypes = GetOperateOnTypes(operateOnTypesAsString); if (operateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments)) { var editRevisions = new EditRevisionsConfigurationCommand(new RevisionsConfiguration { Collections = new Dictionary <string, RevisionsCollectionConfiguration> { ["Orders"] = new RevisionsCollectionConfiguration { Disabled = false } } }, Database.Name, GetRaftRequestIdFromQuery() + "/revisions"); var(index, _) = await ServerStore.SendToLeaderAsync(editRevisions); await Database.RachisLogIndexNotifications.WaitForIndexNotification(index, Database.ServerStore.Engine.OperationTimeout); } if (operateOnTypes.HasFlag(DatabaseItemType.TimeSeries)) { var tsConfig = new TimeSeriesConfiguration { NamedValues = new Dictionary <string, Dictionary <string, string[]> > { ["Companies"] = new Dictionary <string, string[]> { ["StockPrices"] = new[] { "Open", "Close", "High", "Low", "Volume" } }, ["Employees"] = new Dictionary <string, string[]> { ["HeartRates"] = new[] { "BPM" } } } }; var editTimeSeries = new EditTimeSeriesConfigurationCommand(tsConfig, Database.Name, GetRaftRequestIdFromQuery() + "/time-series"); var(index, _) = await ServerStore.SendToLeaderAsync(editTimeSeries); await Database.RachisLogIndexNotifications.WaitForIndexNotification(index, Database.ServerStore.Engine.OperationTimeout); } await using (var sampleData = typeof(SampleDataHandler).Assembly .GetManifestResourceStream("Raven.Server.Web.Studio.EmbeddedData.Northwind.ravendbdump")) { await using (var stream = new GZipStream(sampleData, CompressionMode.Decompress)) using (var source = new StreamSource(stream, context, Database)) { var destination = new DatabaseDestination(Database); var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options: new DatabaseSmugglerOptionsServerSide { OperateOnTypes = operateOnTypes, SkipRevisionCreation = true }); await smuggler.ExecuteAsync(); } } await NoContent(); }