private async Task ImportIncrementalData(CounterConnectionStringOptions connectionString, Stream stream) { CountingStream sizeStream; JsonTextReader jsonReader; if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false) { throw new InvalidOperationException("Failed to get reader for the data stream."); } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } ICounterStore store = null; try { if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } store = new CounterStore { Url = connectionString.Url, Name = connectionString.CounterStoreId, Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials) }; store.Initialize(true); ShowProgress($"Initialized connection to counter store (name = {store.Name})"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { if (jsonReader.TokenType != JsonToken.StartObject) { continue; } var counterDelta = RavenJToken.ReadFrom(jsonReader).ToObject <CounterState>(); ShowProgress($"Importing counter {counterDelta.GroupName} - {counterDelta.CounterName}"); if (counterDelta.Sign == ValueSign.Negative) { counterDelta.Value = -counterDelta.Value; } store.Batch.ScheduleChange(counterDelta.GroupName, counterDelta.CounterName, counterDelta.Value); } ShowProgress("Finished import of the current file."); await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false); } finally { store?.Dispose(); } }
private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = databaseOptions.Limit - totalCount; if (maxRecords > 0) { var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords); using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false)) { while (await documents.MoveNextAsync().ConfigureAwait(false)) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id"))); lastEtag = tempLastEtag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer); if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0) { lastEtag = databaseStatistics.LastDocEtag; } ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName); if (doc != null) { doc.Metadata["@id"] = doc.Key; var jsonDoc = doc.ToJson(); AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc))); totalCount++; } }); } }); await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); try { while (true) { try { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) { continue; } metadata = document.Value <RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) { timeout = 30; } try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) { return(Etag.Empty); } if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) { return; } doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
//assumes that the caller has responsibility to handle data stream disposal ("stream" parameter) private async Task ImportFullData(CounterConnectionStringOptions connectionString, Stream stream) { CountingStream sizeStream; JsonTextReader jsonReader; if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false) { throw new InvalidOperationException("Failed to get reader for the data stream."); } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } ICounterStore store = null; try { if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } store = new CounterStore { Url = connectionString.Url, Name = connectionString.CounterStoreId, Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials) }; store.Initialize(true); ShowProgress($"Initialized connection to counter store (name = {store.Name})"); var existingCounterGroupsAndNames = await store.Admin.GetCounterStorageNameAndGroups(token : CancellationToken) .WithCancellation(CancellationToken) .ConfigureAwait(false); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { if (jsonReader.TokenType != JsonToken.StartObject) { continue; } var counterInfo = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var delta = Math.Abs(counterInfo.Value <long>("Positive")) - Math.Abs(counterInfo.Value <long>("Negative")); var groupName = counterInfo.Value <string>("Group"); var counterName = counterInfo.Value <string>("Name"); if (existingCounterGroupsAndNames.Any(x => x.Group == groupName && x.Name == counterName)) { ShowProgress($"Counter {groupName} - {counterName} is already there. Reset is performed"); await store.ResetAsync(groupName, counterName, CancellationToken).ConfigureAwait(false); //since it is a full import, the values are overwritten } ShowProgress($"Importing counter {groupName} - {counterName}"); store.Batch.ScheduleChange(groupName, counterName, delta); } ShowProgress("Finished import..."); await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false); } finally { store?.Dispose(); } }
public override async Task SmuggleAsync(DatabaseSmugglerOperationState state, CancellationToken cancellationToken) { using (var actions = Destination.DocumentActions()) { if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) == false) { await Source.SkipDocumentsAsync(cancellationToken).ConfigureAwait(false); return; } var afterEtag = state.LastDocsEtag; var maxEtag = _maxEtags.LastDocsEtag; var now = SystemTime.UtcNow; var totalCount = 0; var reachedMaxEtag = false; var affectedCollections = new List <string>(); Options.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { affectedCollections.Add(collectionName); }); } }); do { var hasDocs = false; try { var maxRecords = Options.Limit - totalCount; if (maxRecords > 0 && reachedMaxEtag == false) { var pageSize = Source.SupportsPaging ? Math.Min(Options.BatchSize, maxRecords) : int.MaxValue; using (var documents = await Source.ReadDocumentsAfterAsync(afterEtag, pageSize, cancellationToken).ConfigureAwait(false)) { while (await documents.MoveNextAsync().ConfigureAwait(false)) { hasDocs = true; var currentDocument = documents.Current; var currentEtag = Etag.Parse(currentDocument.Value <RavenJObject>("@metadata").Value <string>("@etag")); var currentKey = currentDocument["@metadata"].Value <string>("@id"); Notifications.OnDocumentRead(this, currentKey); if (maxEtag != null && currentEtag.CompareTo(maxEtag) > 0) { reachedMaxEtag = true; break; } afterEtag = currentEtag; if (Options.MatchFilters(currentDocument) == false) { if (affectedCollections.Count <= 0) { continue; } if (currentDocument.ContainsKey("@metadata") == false) { continue; } if (currentKey == null) { continue; } var isHilo = currentKey.StartsWith("Raven/Hilo/", StringComparison.OrdinalIgnoreCase); if (isHilo && Source.SupportsReadingHiLoDocuments) { continue; } if (isHilo == false || affectedCollections.Any(x => currentKey.EndsWith("/" + x, StringComparison.OrdinalIgnoreCase)) == false) { continue; } } if (Options.ShouldExcludeExpired && Options.ExcludeExpired(currentDocument, now)) { continue; } if (string.IsNullOrEmpty(Options.TransformScript) == false) { currentDocument = await TransformDocumentAsync(currentDocument).ConfigureAwait(false); } // If document is null after a transform we skip it. if (currentDocument == null) { continue; } var metadata = currentDocument["@metadata"] as RavenJObject; if (metadata != null) { if (Options.SkipConflicted && metadata.ContainsKey(Constants.RavenReplicationConflictDocument)) { continue; } if (Options.StripReplicationInformation) { currentDocument["@metadata"] = SmugglerHelper.StripReplicationInformationFromMetadata(metadata); //TODO arek - why there three methods are in SmugglerHelper, they are used just here } if (Options.ShouldDisableVersioningBundle) { currentDocument["@metadata"] = SmugglerHelper.DisableVersioning(metadata); } currentDocument["@metadata"] = SmugglerHelper.HandleConflictDocuments(metadata); } try { await actions.WriteDocumentAsync(currentDocument, cancellationToken).ConfigureAwait(false); Notifications.OnDocumentWrite(this, currentKey); } catch (Exception e) { if (Options.IgnoreErrorsAndContinue == false) { throw; } Notifications.ShowProgress(DatabaseSmugglerMessages.Documents_Write_Failure, currentKey, e.Message); } totalCount++; if (totalCount % Options.BatchSize == 0) { await SaveOperationStateAsync(state, currentEtag, cancellationToken).ConfigureAwait(false); // Wait for the batch to be indexed before continue. if (Destination.SupportsWaitingForIndexing) { await Destination.WaitForIndexingAsOfLastWriteAsync(cancellationToken).ConfigureAwait(false); } } } } if (hasDocs) { continue; } } if (Source.SupportsReadingHiLoDocuments) { // Load HiLo documents for selected collections foreach (var filter in Options.Filters) { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase) == false) { continue; } foreach (var collectionName in filter.Values) { var doc = await Source .ReadDocumentAsync("Raven/HiLo/" + collectionName, cancellationToken) .ConfigureAwait(false); if (doc == null) { continue; } await actions.WriteDocumentAsync(doc, cancellationToken).ConfigureAwait(false); totalCount++; } } } await SaveOperationStateAsync(state, afterEtag, cancellationToken).ConfigureAwait(false); state.LastDocsEtag = afterEtag; return; } catch (Exception e) { throw new SmugglerException(e.Message, e) { LastEtag = afterEtag }; } } while (Source.SupportsPaging); } }
private static async Task <Tuple <int, Etag, DateTime> > TransferStreamedDocuments(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, DateTime now, SmugglerJintHelper jintHelper, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { Etag lastReadEtag = fromEtag; using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastReadEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) { continue; } metadata = document.Value <RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } return(Tuple.Create(totalCount, lastReadEtag, lastReport)); }
private static async Task <Tuple <int, string, DateTime> > TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); fromEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } return(Tuple.Create(totalCount, fromEtag, lastReport)); }