private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = databaseOptions.Limit - totalCount; if (maxRecords > 0) { var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords); using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false)) { while (await documents.MoveNextAsync().ConfigureAwait(false)) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id"))); lastEtag = tempLastEtag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer); if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0) { lastEtag = databaseStatistics.LastDocEtag; } ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName); if (doc != null) { doc.Metadata["@id"] = doc.Key; var jsonDoc = doc.ToJson(); AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc))); totalCount++; } }); } }); await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); try { while (true) { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); document.Remove("@metadata"); bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (RavenJObject document in documents) { var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } break; } catch (Exception e) { if (retries-- == 0) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } } finally { bulkInsertOperation.Dispose(); } }
private static async Task<Tuple<int, Etag, DateTime>> TransferStreamedDocuments(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, DateTime now, SmugglerJintHelper jintHelper, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { Etag lastReadEtag = fromEtag; using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastReadEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) continue; metadata = document.Value<RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } return Tuple.Create(totalCount, lastReadEtag, lastReport); }
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); try { while (true) { try { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) { continue; } metadata = document.Value <RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) { timeout = 30; } try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) { return(Etag.Empty); } if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) { return; } doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
private static async Task<Tuple<int, string, DateTime>> TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { var documents = await ((AsyncServerClient) exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject) jToken; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); fromEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } return Tuple.Create(totalCount, fromEtag, lastReport); }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); try { while (true) { try { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) continue; metadata = document.Value<RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.Empty; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) return; doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); try { while (true) { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); document.Remove("@metadata"); bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) { timeout = 30; } try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (RavenJObject document in documents) { var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } break; } catch (Exception e) { if (retries-- == 0) { throw; } exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return(lastEtag); } } finally { bulkInsertOperation.Dispose(); } }
private static async Task <Tuple <int, Etag, DateTime> > TransferStreamedDocuments(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, DateTime now, SmugglerJintHelper jintHelper, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { Etag lastReadEtag = fromEtag; using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastReadEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) { continue; } metadata = document.Value <RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } return(Tuple.Create(totalCount, lastReadEtag, lastReport)); }
private static async Task <Tuple <int, string, DateTime> > TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport) { var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); fromEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } return(Tuple.Create(totalCount, fromEtag, lastReport)); }