private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions)
        {
            var now = SystemTime.UtcNow;

            string lastEtag       = databaseOptions.StartDocsEtag;
            var    totalCount     = 0;
            var    lastReport     = SystemTime.UtcNow;
            var    reportInterval = TimeSpan.FromSeconds(2);

            ShowProgress("Exporting Documents");

            while (true)
            {
                bool hasDocs = false;
                try
                {
                    var maxRecords = databaseOptions.Limit - totalCount;
                    if (maxRecords > 0)
                    {
                        var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords);
                        using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false))
                        {
                            while (await documents.MoveNextAsync().ConfigureAwait(false))
                            {
                                hasDocs = true;
                                var document = documents.Current;

                                var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag"));

                                Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id")));

                                lastEtag = tempLastEtag;

                                if (!databaseOptions.MatchFilters(document))
                                {
                                    continue;
                                }

                                if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                {
                                    continue;
                                }

                                if (databaseOptions.StripReplicationInformation)
                                {
                                    document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                                }

                                if (databaseOptions.ShouldDisableVersioningBundle)
                                {
                                    document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject);
                                }

                                document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                                await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false);

                                totalCount++;

                                if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                {
                                    ShowProgress("Exported {0} documents", totalCount);
                                    lastReport = SystemTime.UtcNow;
                                }
                            }
                        }

                        if (hasDocs)
                        {
                            continue;
                        }

                        // The server can filter all the results. In this case, we need to try to go over with the next batch.
                        // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
                        // (This code provides support for legacy RavenDB version: 1.0)
                        var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false);

                        var lastEtagComparable = new ComparableByteArray(lastEtag);
                        if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
                        {
                            lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer);
                            if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0)
                            {
                                lastEtag = databaseStatistics.LastDocEtag;
                            }
                            ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
                            continue;
                        }
                    }
                }
                catch (Exception e)
                {
                    ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
                    ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                    throw new SmugglerExportException(e.Message, e)
                          {
                              LastEtag = lastEtag,
                          };
                }

                // Load HiLo documents for selected collections
                databaseOptions.Filters.ForEach(filter =>
                {
                    if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
                    {
                        filter.Values.ForEach(collectionName =>
                        {
                            JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName);
                            if (doc != null)
                            {
                                doc.Metadata["@id"] = doc.Key;
                                var jsonDoc         = doc.ToJson();
                                AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc)));
                                totalCount++;
                            }
                        });
                    }
                });

                await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush

                ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                return(lastEtag);
            }
        }
		private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
		{
			var now = SystemTime.UtcNow;

			string lastEtag = databaseOptions.StartDocsEtag;
			var totalCount = 0;
			var lastReport = SystemTime.UtcNow;
			var reportInterval = TimeSpan.FromSeconds(2);
			ShowProgress("Exporting Documents");

			var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
			                                                       {
																	   BatchSize = importBatchSize,
				                                                       OverwriteExisting = true,
			                                                       });
			bulkInsertOperation.Report += text => ShowProgress(text);

			try
			{
				while (true)
				{
					if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
					{
						ShowProgress("Streaming documents from " + lastEtag);
						using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
						{
							while (await documentsEnumerator.MoveNextAsync())
							{
								var document = documentsEnumerator.Current;

								if (!databaseOptions.MatchFilters(document))
									continue;
								if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
									continue;

								var metadata = document.Value<RavenJObject>("@metadata");
								var id = metadata.Value<string>("@id");
								var etag = Etag.Parse(metadata.Value<string>("@etag"));
								document.Remove("@metadata");
								bulkInsertOperation.Store(document, metadata, id);
								totalCount++;

								if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
								{
									ShowProgress("Exported {0} documents", totalCount);
									lastReport = SystemTime.UtcNow;
								}

								lastEtag = etag;
							}
						}
					}
					else
					{
						int retries = RetriesCount;
						var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
						var timeout = databaseOptions.Timeout.Seconds;
						if (timeout < 30)
							timeout = 30;
						try
						{
							var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);

							while (true)
							{
								try
								{
									ShowProgress("Get documents from " + lastEtag);
									var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
									foreach (RavenJObject document in documents)
									{
										var metadata = document.Value<RavenJObject>("@metadata");
										var id = metadata.Value<string>("@id");
										var etag = Etag.Parse(metadata.Value<string>("@etag"));
										document.Remove("@metadata");
										metadata.Remove("@id");
										metadata.Remove("@etag");

										if (!databaseOptions.MatchFilters(document))
											continue;
										if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
											continue;

										bulkInsertOperation.Store(document, metadata, id);
										totalCount++;

										if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
										{
											ShowProgress("Exported {0} documents", totalCount);
											lastReport = SystemTime.UtcNow;
										}
										lastEtag = etag;
									}
									break;
								}
								catch (Exception e)
								{
									if (retries-- == 0)
										throw;
									exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
									importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
									ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
								}
							}
						}
						finally
						{
							exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
						}
					}

					// In a case that we filter all the results, the formEtag hasn't updaed to the latest, 
					// but we still need to continue until we finish all the docs.
					var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
					var lastEtagComparable = new ComparableByteArray(lastEtag);
					if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
					{
						lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
						ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
						continue;
					}

					ShowProgress("Done with reading documents, total: {0}", totalCount);
					return lastEtag;
				}
			}
			finally
			{
				bulkInsertOperation.Dispose();
			}
		}
        private static async Task<Tuple<int, Etag, DateTime>> TransferStreamedDocuments(DocumentStore exportStore, 
            SmugglerDatabaseOptions databaseOptions, 
            DateTime now, 
            SmugglerJintHelper jintHelper, 
            BulkInsertOperation bulkInsertOperation, 
            TimeSpan reportInterval, 
            int totalCount, 
            string fromEtag, 
            DateTime lastReport)
        {
            Etag lastReadEtag = fromEtag;
            using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
            {
                while (await documentsEnumerator.MoveNextAsync())
                {
                    var document = documentsEnumerator.Current;
                    var metadata = document.Value<RavenJObject>("@metadata");
                    var id = metadata.Value<string>("@id");
                    var etag = Etag.Parse(metadata.Value<string>("@etag"));

                    lastReadEtag = etag;

                    if (!databaseOptions.MatchFilters(document))
                        continue;
                    if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                        continue;

                    if (databaseOptions.StripReplicationInformation)
                        document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);

                    if (databaseOptions.ShouldDisableVersioningBundle)
                        document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);

                    document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                    if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
                    {
                        document = jintHelper.Transform(databaseOptions.TransformScript, document);
                        if (document == null)
                            continue;
                        metadata = document.Value<RavenJObject>("@metadata");
                    }

                    document.Remove("@metadata");
                    try
                    {
                        bulkInsertOperation.Store(document, metadata, id);
                    }
                    catch (Exception e)
                    {
                        if (databaseOptions.IgnoreErrorsAndContinue == false)
                            throw;

                        ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                    }

                    totalCount++;

                    if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                    {
                        ShowProgress("Exported {0} documents", totalCount);
                        lastReport = SystemTime.UtcNow;
                    }
                }
            }
            return Tuple.Create(totalCount, lastReadEtag, lastReport);
        }
Beispiel #4
0
        private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
        {
            var now = SystemTime.UtcNow;

            string lastEtag       = databaseOptions.StartDocsEtag;
            var    totalCount     = 0;
            var    lastReport     = SystemTime.UtcNow;
            var    reportInterval = TimeSpan.FromSeconds(2);

            ShowProgress("Exporting Documents");

            var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
            {
                BatchSize         = importBatchSize,
                OverwriteExisting = true,
            });

            bulkInsertOperation.Report += text => ShowProgress(text);
            var jintHelper = new SmugglerJintHelper();

            jintHelper.Initialize(databaseOptions);
            try
            {
                while (true)
                {
                    try
                    {
                        if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
                        {
                            ShowProgress("Streaming documents from " + lastEtag);
                            using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
                            {
                                while (await documentsEnumerator.MoveNextAsync())
                                {
                                    var document = documentsEnumerator.Current;
                                    var metadata = document.Value <RavenJObject>("@metadata");
                                    var id       = metadata.Value <string>("@id");
                                    var etag     = Etag.Parse(metadata.Value <string>("@etag"));

                                    lastEtag = etag;

                                    if (!databaseOptions.MatchFilters(document))
                                    {
                                        continue;
                                    }
                                    if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                    {
                                        continue;
                                    }

                                    if (databaseOptions.StripReplicationInformation)
                                    {
                                        document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                                    }

                                    if (databaseOptions.ShouldDisableVersioningBundle)
                                    {
                                        document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
                                    }

                                    document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                                    if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
                                    {
                                        document = jintHelper.Transform(databaseOptions.TransformScript, document);
                                        if (document == null)
                                        {
                                            continue;
                                        }
                                        metadata = document.Value <RavenJObject>("@metadata");
                                    }

                                    document.Remove("@metadata");
                                    try
                                    {
                                        bulkInsertOperation.Store(document, metadata, id);
                                    }
                                    catch (Exception e)
                                    {
                                        if (databaseOptions.IgnoreErrorsAndContinue == false)
                                        {
                                            throw;
                                        }

                                        ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                                    }

                                    totalCount++;

                                    if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                    {
                                        ShowProgress("Exported {0} documents", totalCount);
                                        lastReport = SystemTime.UtcNow;
                                    }
                                }
                            }
                        }
                        else
                        {
                            int retries = RetriesCount;
                            var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
                            var timeout = databaseOptions.Timeout.Seconds;
                            if (timeout < 30)
                            {
                                timeout = 30;
                            }
                            try
                            {
                                var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);

                                while (true)
                                {
                                    try
                                    {
                                        ShowProgress("Get documents from " + lastEtag);
                                        var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
                                        foreach (var jToken in documents)
                                        {
                                            var document = (RavenJObject)jToken;
                                            var metadata = document.Value <RavenJObject>("@metadata");
                                            var id       = metadata.Value <string>("@id");
                                            var etag     = Etag.Parse(metadata.Value <string>("@etag"));
                                            lastEtag = etag;

                                            if (!databaseOptions.MatchFilters(document))
                                            {
                                                continue;
                                            }
                                            if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                            {
                                                continue;
                                            }

                                            if (databaseOptions.StripReplicationInformation)
                                            {
                                                document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                                            }

                                            if (databaseOptions.ShouldDisableVersioningBundle)
                                            {
                                                document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
                                            }

                                            document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                                            document.Remove("@metadata");
                                            metadata.Remove("@id");
                                            metadata.Remove("@etag");

                                            try
                                            {
                                                bulkInsertOperation.Store(document, metadata, id);
                                            }
                                            catch (Exception e)
                                            {
                                                if (databaseOptions.IgnoreErrorsAndContinue == false)
                                                {
                                                    throw;
                                                }

                                                ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                                            }

                                            totalCount++;

                                            if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                            {
                                                ShowProgress("Exported {0} documents", totalCount);
                                                lastReport = SystemTime.UtcNow;
                                            }
                                        }
                                        break;
                                    }
                                    catch (Exception e)
                                    {
                                        if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
                                        {
                                            return(Etag.Empty);
                                        }

                                        if (databaseOptions.IgnoreErrorsAndContinue == false)
                                        {
                                            throw;
                                        }

                                        exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
                                        importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
                                        ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
                                    }
                                }
                            }
                            finally
                            {
                                exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
                            }
                        }

                        // In a case that we filter all the results, the formEtag hasn't updaed to the latest,
                        // but we still need to continue until we finish all the docs.
                        var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();

                        var lastEtagComparable = new ComparableByteArray(lastEtag);
                        if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
                        {
                            lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
                            ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
                            continue;
                        }

                        // Load HiLo documents for selected collections
                        databaseOptions.Filters.ForEach(filter =>
                        {
                            if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
                            {
                                filter.Values.ForEach(collectionName =>
                                {
                                    var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
                                    if (doc == null)
                                    {
                                        return;
                                    }

                                    doc.Metadata["@id"] = doc.Key;
                                    bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
                                    totalCount++;
                                });
                            }
                        });

                        ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                        return(lastEtag);
                    }
                    catch (Exception e)
                    {
                        ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
                        ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                        throw new SmugglerExportException(e.Message, e)
                              {
                                  LastEtag = lastEtag,
                              };
                    }
                }
            }
            finally
            {
                bulkInsertOperation.Dispose();
            }
        }
        private static async Task<Tuple<int, string, DateTime>> TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
        {
            var documents = await ((AsyncServerClient) exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);
            foreach (var jToken in documents)
            {
                var document = (RavenJObject) jToken;
                var metadata = document.Value<RavenJObject>("@metadata");
                var id = metadata.Value<string>("@id");
                var etag = Etag.Parse(metadata.Value<string>("@etag"));
                fromEtag = etag;

                if (!databaseOptions.MatchFilters(document))
                    continue;
                if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                    continue;

                if (databaseOptions.StripReplicationInformation)
                    document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);

                if (databaseOptions.ShouldDisableVersioningBundle)
                    document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);

                document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                document.Remove("@metadata");
                metadata.Remove("@id");
                metadata.Remove("@etag");

                try
                {
                    bulkInsertOperation.Store(document, metadata, id);
                }
                catch (Exception e)
                {
                    if (databaseOptions.IgnoreErrorsAndContinue == false)
                        throw;

                    ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                }

                totalCount++;

                if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                {
                    ShowProgress("Exported {0} documents", totalCount);
                    lastReport = SystemTime.UtcNow;
                }
            }
            return Tuple.Create(totalCount, fromEtag, lastReport);
        }
        private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
        {
            var now = SystemTime.UtcNow;

            string lastEtag = databaseOptions.StartDocsEtag;
            var totalCount = 0;
            var lastReport = SystemTime.UtcNow;
            var reportInterval = TimeSpan.FromSeconds(2);
            ShowProgress("Exporting Documents");

            var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
            {
                BatchSize = importBatchSize,
                OverwriteExisting = true,
            });
            bulkInsertOperation.Report += text => ShowProgress(text);
            var jintHelper = new SmugglerJintHelper();
            jintHelper.Initialize(databaseOptions);
            try
            {
                while (true)
                {
                    try
                    {
                        if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
                        {
                            ShowProgress("Streaming documents from " + lastEtag);
                            using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
                            {
                                while (await documentsEnumerator.MoveNextAsync())
                                {
                                    var document = documentsEnumerator.Current;
                                    var metadata = document.Value<RavenJObject>("@metadata");
                                    var id = metadata.Value<string>("@id");
                                    var etag = Etag.Parse(metadata.Value<string>("@etag"));

                                    lastEtag = etag;

                                    if (!databaseOptions.MatchFilters(document))
                                        continue;
                                    if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                        continue;

                                    if (databaseOptions.StripReplicationInformation)
                                        document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);

                                    if (databaseOptions.ShouldDisableVersioningBundle)
                                        document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);

                                    document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                                    if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
                                    {
                                        document = jintHelper.Transform(databaseOptions.TransformScript, document);
                                        if (document == null)
                                            continue;
                                        metadata = document.Value<RavenJObject>("@metadata");
                                    }

                                    document.Remove("@metadata");
                                    try
                                    {
                                        bulkInsertOperation.Store(document, metadata, id);
                                    }
                                    catch (Exception e)
                                    {
                                        if (databaseOptions.IgnoreErrorsAndContinue == false)
                                            throw;

                                        ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                                    }

                                    totalCount++;

                                    if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                    {
                                        ShowProgress("Exported {0} documents", totalCount);
                                        lastReport = SystemTime.UtcNow;
                                    }
                                }
                            }
                        }
                        else
                        {
                            int retries = RetriesCount;
                            var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
                            var timeout = databaseOptions.Timeout.Seconds;
                            if (timeout < 30)
                                timeout = 30;
                            try
                            {
                                var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);

                                while (true)
                                {
                                    try
                                    {
                                        ShowProgress("Get documents from " + lastEtag);
                                        var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
                                        foreach (var jToken in documents)
                                        {
                                            var document = (RavenJObject)jToken;
                                            var metadata = document.Value<RavenJObject>("@metadata");
                                            var id = metadata.Value<string>("@id");
                                            var etag = Etag.Parse(metadata.Value<string>("@etag"));
                                            lastEtag = etag;

                                            if (!databaseOptions.MatchFilters(document))
                                                continue;
                                            if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                                continue;

                                            if (databaseOptions.StripReplicationInformation)
                                                document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);

                                            if (databaseOptions.ShouldDisableVersioningBundle)
                                                document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);

                                            document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                                            document.Remove("@metadata");
                                            metadata.Remove("@id");
                                            metadata.Remove("@etag");

                                            try
                                            {
                                                bulkInsertOperation.Store(document, metadata, id);
                                            }
                                            catch (Exception e)
                                            {
                                                if (databaseOptions.IgnoreErrorsAndContinue == false)
                                                    throw;

                                                ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                                            }

                                            totalCount++;

                                            if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                            {
                                                ShowProgress("Exported {0} documents", totalCount);
                                                lastReport = SystemTime.UtcNow;
                                            }
                                        }
                                        break;
                                    }
                                    catch (Exception e)
                                    {
                                        if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue)
                                            return Etag.Empty;

                                        if (databaseOptions.IgnoreErrorsAndContinue == false)
                                            throw;

                                        exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
                                        importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
                                        ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
                                    }
                                }
                            }
                            finally
                            {
                                exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
                            }
                        }

                        // In a case that we filter all the results, the formEtag hasn't updaed to the latest, 
                        // but we still need to continue until we finish all the docs.
                        var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();
                        var lastEtagComparable = new ComparableByteArray(lastEtag);
                        if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
                        {
                            lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
                            ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
                            continue;
                        }

                        // Load HiLo documents for selected collections
                        databaseOptions.Filters.ForEach(filter =>
                        {
                            if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase))
                            {
                                filter.Values.ForEach(collectionName =>
                                {
                                    var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName);
                                    if (doc == null)
                                        return;

                                    doc.Metadata["@id"] = doc.Key;
                                    bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key);
                                    totalCount++;
                                });
                            }
                        });

                        ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                        return lastEtag;
                    }
                    catch (Exception e)
                    {
                        ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message);
                        ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                        throw new SmugglerExportException(e.Message, e)
                        {
                            LastEtag = lastEtag,
                        };
                    }
                }
            }
            finally
            {
                bulkInsertOperation.Dispose();
            }
        }
        private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
        {
            var now = SystemTime.UtcNow;

            string lastEtag       = databaseOptions.StartDocsEtag;
            var    totalCount     = 0;
            var    lastReport     = SystemTime.UtcNow;
            var    reportInterval = TimeSpan.FromSeconds(2);

            ShowProgress("Exporting Documents");

            var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
            {
                BatchSize         = importBatchSize,
                OverwriteExisting = true,
            });

            bulkInsertOperation.Report += text => ShowProgress(text);

            try
            {
                while (true)
                {
                    if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
                    {
                        ShowProgress("Streaming documents from " + lastEtag);
                        using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
                        {
                            while (await documentsEnumerator.MoveNextAsync())
                            {
                                var document = documentsEnumerator.Current;

                                if (!databaseOptions.MatchFilters(document))
                                {
                                    continue;
                                }
                                if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                {
                                    continue;
                                }

                                if (databaseOptions.StripReplicationInformation)
                                {
                                    document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                                }

                                var metadata = document.Value <RavenJObject>("@metadata");
                                var id       = metadata.Value <string>("@id");
                                var etag     = Etag.Parse(metadata.Value <string>("@etag"));
                                document.Remove("@metadata");
                                bulkInsertOperation.Store(document, metadata, id);
                                totalCount++;

                                if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                {
                                    ShowProgress("Exported {0} documents", totalCount);
                                    lastReport = SystemTime.UtcNow;
                                }

                                lastEtag = etag;
                            }
                        }
                    }
                    else
                    {
                        int retries = RetriesCount;
                        var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
                        var timeout = databaseOptions.Timeout.Seconds;
                        if (timeout < 30)
                        {
                            timeout = 30;
                        }
                        try
                        {
                            var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);

                            while (true)
                            {
                                try
                                {
                                    ShowProgress("Get documents from " + lastEtag);
                                    var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
                                    foreach (RavenJObject document in documents)
                                    {
                                        var metadata = document.Value <RavenJObject>("@metadata");
                                        var id       = metadata.Value <string>("@id");
                                        var etag     = Etag.Parse(metadata.Value <string>("@etag"));
                                        document.Remove("@metadata");
                                        metadata.Remove("@id");
                                        metadata.Remove("@etag");

                                        if (!databaseOptions.MatchFilters(document))
                                        {
                                            continue;
                                        }
                                        if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                                        {
                                            continue;
                                        }

                                        if (databaseOptions.StripReplicationInformation)
                                        {
                                            document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                                        }

                                        bulkInsertOperation.Store(document, metadata, id);
                                        totalCount++;

                                        if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                                        {
                                            ShowProgress("Exported {0} documents", totalCount);
                                            lastReport = SystemTime.UtcNow;
                                        }
                                        lastEtag = etag;
                                    }
                                    break;
                                }
                                catch (Exception e)
                                {
                                    if (retries-- == 0)
                                    {
                                        throw;
                                    }
                                    exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
                                    importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
                                    ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e);
                                }
                            }
                        }
                        finally
                        {
                            exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout;
                        }
                    }

                    // In a case that we filter all the results, the formEtag hasn't updaed to the latest,
                    // but we still need to continue until we finish all the docs.
                    var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync();

                    var lastEtagComparable = new ComparableByteArray(lastEtag);
                    if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
                    {
                        lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize);
                        ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);
                        continue;
                    }

                    ShowProgress("Done with reading documents, total: {0}", totalCount);
                    return(lastEtag);
                }
            }
            finally
            {
                bulkInsertOperation.Dispose();
            }
        }
Beispiel #8
0
        private static async Task <Tuple <int, Etag, DateTime> > TransferStreamedDocuments(DocumentStore exportStore,
                                                                                           SmugglerDatabaseOptions databaseOptions,
                                                                                           DateTime now,
                                                                                           SmugglerJintHelper jintHelper,
                                                                                           BulkInsertOperation bulkInsertOperation,
                                                                                           TimeSpan reportInterval,
                                                                                           int totalCount,
                                                                                           string fromEtag,
                                                                                           DateTime lastReport)
        {
            Etag lastReadEtag = fromEtag;

            using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(fromEtag))
            {
                while (await documentsEnumerator.MoveNextAsync())
                {
                    var document = documentsEnumerator.Current;
                    var metadata = document.Value <RavenJObject>("@metadata");
                    var id       = metadata.Value <string>("@id");
                    var etag     = Etag.Parse(metadata.Value <string>("@etag"));

                    lastReadEtag = etag;

                    if (!databaseOptions.MatchFilters(document))
                    {
                        continue;
                    }
                    if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                    {
                        continue;
                    }

                    if (databaseOptions.StripReplicationInformation)
                    {
                        document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                    }

                    if (databaseOptions.ShouldDisableVersioningBundle)
                    {
                        document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
                    }

                    document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                    if (!string.IsNullOrEmpty(databaseOptions.TransformScript))
                    {
                        document = jintHelper.Transform(databaseOptions.TransformScript, document);
                        if (document == null)
                        {
                            continue;
                        }
                        metadata = document.Value <RavenJObject>("@metadata");
                    }

                    document.Remove("@metadata");
                    try
                    {
                        bulkInsertOperation.Store(document, metadata, id);
                    }
                    catch (Exception e)
                    {
                        if (databaseOptions.IgnoreErrorsAndContinue == false)
                        {
                            throw;
                        }

                        ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                    }

                    totalCount++;

                    if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                    {
                        ShowProgress("Exported {0} documents", totalCount);
                        lastReport = SystemTime.UtcNow;
                    }
                }
            }
            return(Tuple.Create(totalCount, lastReadEtag, lastReport));
        }
Beispiel #9
0
        private static async Task <Tuple <int, string, DateTime> > TransferDocumentsWithoutStreaming(DocumentStore exportStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize, OperationMetadata operationMetadata, DateTime now, BulkInsertOperation bulkInsertOperation, TimeSpan reportInterval, int totalCount, string fromEtag, DateTime lastReport)
        {
            var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, fromEtag, exportBatchSize, operationMetadata);

            foreach (var jToken in documents)
            {
                var document = (RavenJObject)jToken;
                var metadata = document.Value <RavenJObject>("@metadata");
                var id       = metadata.Value <string>("@id");
                var etag     = Etag.Parse(metadata.Value <string>("@etag"));
                fromEtag = etag;

                if (!databaseOptions.MatchFilters(document))
                {
                    continue;
                }
                if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now))
                {
                    continue;
                }

                if (databaseOptions.StripReplicationInformation)
                {
                    document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject);
                }

                if (databaseOptions.ShouldDisableVersioningBundle)
                {
                    document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject);
                }

                document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject);

                document.Remove("@metadata");
                metadata.Remove("@id");
                metadata.Remove("@etag");

                try
                {
                    bulkInsertOperation.Store(document, metadata, id);
                }
                catch (Exception e)
                {
                    if (databaseOptions.IgnoreErrorsAndContinue == false)
                    {
                        throw;
                    }

                    ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message);
                }

                totalCount++;

                if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
                {
                    ShowProgress("Exported {0} documents", totalCount);
                    lastReport = SystemTime.UtcNow;
                }
            }
            return(Tuple.Create(totalCount, fromEtag, lastReport));
        }