public void CleanupDocuments(Etag lastIndexedEtag) { var highest = new ComparableByteArray(lastIndexedEtag); foreach (var docToRemove in documentsToRemove) { if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false) { continue; } HashSet <Etag> _; documentsToRemove.TryRemove(docToRemove.Key, out _); } foreach (var updatedDocs in updatedDocuments) { if (updatedDocs.Value.All(etag => highest.CompareTo(etag) > 0) == false) { continue; } HashSet <Etag> _; updatedDocuments.TryRemove(updatedDocs.Key, out _); } JsonDocument result; while (prefetchingQueue.TryPeek(out result) && highest.CompareTo(result.Etag) >= 0) { prefetchingQueue.TryDequeue(out result); } }
private IEnumerable <Tuple <IndexToWorkOn, IEnumerable <JsonDocument> > > FilterIndexes(IEnumerable <IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); Action <IStorageActionsAccessor> action = null; foreach (var indexToWorkOn in indexesToWorkOn) { var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()); if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0) { continue; } var filteredDocs = jsonDocs.Where(doc => indexLastInedexEtag.CompareTo(new ComparableByteArray(doc.Etag.Value.ToByteArray())) < 0); var indexName = indexToWorkOn.IndexName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if (viewGenerator == null) { continue; // probably deleted } if (viewGenerator.ForEntityNames.Count != 0) // limit for the items that it care for { filteredDocs = filteredDocs.Where(x => viewGenerator.ForEntityNames.Contains(x.Metadata.Value <string>(Constants.RavenEntityName))); } List <JsonDocument> jsonDocuments = filteredDocs.ToList(); if (jsonDocuments.Count == 0) { // we use it this way to batch all the updates together action += accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified); continue; } yield return(Tuple.Create <IndexToWorkOn, IEnumerable <JsonDocument> >(indexToWorkOn, jsonDocuments)); } if (action != null) { transactionalStorage.Batch(action); } }
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag) { var totalCount = 0; while (true) { var array = GetAttachments(totalCount, lastEtag); if (array.Length == 0) { var databaseStatistics = GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += array.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount); foreach (var item in array) { item.WriteTo(jsonWriter); } lastEtag = new Guid(array.Last().Value <string>("Etag")); } }
private bool TryGetInMemoryJsonDocuments(Guid nextDocEtag, List <JsonDocument> items) { if (context.Configuration.DisableDocumentPreFetchingForIndexing) { return(false); } JsonDocument result; bool hasDocs = false; while (inMemoryDocs.TryPeek(out result) && ComparableByteArray.CompareTo(nextDocEtag.ToByteArray(), result.Etag.Value.ToByteArray()) >= 0) { // safe to do peek then dequeue because we are the only one doing the dequeues // and here we are single threaded inMemoryDocs.TryDequeue(out result); if (result.Etag.Value != nextDocEtag) { continue; } items.Add(result); hasDocs = true; nextDocEtag = Etag.Increment(nextDocEtag, 1); } return(hasDocs); }
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag) { var totalCount = 0; while (true) { var array = GetAttachments(totalCount, lastEtag); if (array.Length == 0) { var databaseStatistics = GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += array.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", array.Length, totalCount); foreach (var item in array) { item.WriteTo(jsonWriter); } lastEtag = new Guid(array.Last().Value<string>("Etag")); } }
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { if (maxEtag != null) { throw new ArgumentException("We don't support maxEtag in SmugglerApi", "maxEtag"); } int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; await commands.CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag, "GET") .ReadResponseJsonAsync() .ContinueWith(task => attachmentInfo = (RavenJArray)task.Result); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += attachmentInfo.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; await commands.CreateRequest("/static/" + item.Value <string>("Key"), "GET") .ReadResponseBytesAsync() .ContinueWith(task => attachmentData = task.Result); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); } lastEtag = Etag.Parse(attachmentInfo.Last().Value <string>("Etag")); } }
public void CleanupDocumentsToRemove(Guid lastIndexedEtag) { var highest = new ComparableByteArray(lastIndexedEtag); foreach (var docToRemove in documentsToRemove) { if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false) { continue; } HashSet <Guid> _; documentsToRemove.TryRemove(docToRemove.Key, out _); } JsonDocument result; while (inMemoryDocs.TryPeek(out result) && highest.CompareTo(result.Etag.Value) >= 0) { inMemoryDocs.TryDequeue(out result); } }
protected override async Task<Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag) { int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; await commands.CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag, "GET") .ReadResponseJsonAsync() .ContinueWith(task => attachmentInfo = (RavenJArray) task.Result); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += attachmentInfo.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value<string>("Key")); byte[] attachmentData = null; await commands.CreateRequest("/static/" + item.Value<string>("Key"), "GET") .ReadResponseBytesAsync() .ContinueWith(task => attachmentData = task.Result); new RavenJObject { {"Data", attachmentData}, {"Metadata", item.Value<RavenJObject>("Metadata")}, {"Key", item.Value<string>("Key")} } .WriteTo(jsonWriter); } lastEtag = Etag.Parse(attachmentInfo.Last().Value<string>("Etag")); } }
private static Guid GetLowestEtag(JsonDocument[] past) { var lowest = new ComparableByteArray(past[0].Etag.Value); for (int i = 1; i < past.Length; i++) { var etag = past[i].Etag.Value; if (lowest.CompareTo(etag) < 0) { continue; } lowest = new ComparableByteArray(etag); } return(lowest.ToGuid()); }
public void RemoveAllBefore(string name, Etag etag) { var comparable = new ComparableByteArray(etag); var results = storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject { {"name", name}, {"etag", Guid.Empty.ToByteArray()} }) .TakeWhile(x => String.Equals(x.Value<string>("name"), name, StringComparison.OrdinalIgnoreCase) && comparable.CompareTo(x.Value<byte[]>("etag")) >= 0); foreach (var result in results) { storage.Lists.Remove(result); } }
public void RemoveAllBefore(string name, Guid etag) { var comparable = new ComparableByteArray(etag); var results = storage.Lists["ByNameAndEtag"].SkipAfter(new RavenJObject { { "name", name }, { "etag", Guid.Empty.ToByteArray() } }) .TakeWhile(x => String.Equals(x.Value <string>("name"), name, StringComparison.InvariantCultureIgnoreCase) && comparable.CompareTo(x.Value <byte[]>("etag")) >= 0); foreach (var result in results) { storage.Lists.Remove(result); } }
protected override async Task <Etag> ExportAttachments(JsonTextWriter jsonWriter, Etag lastEtag) { int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; var request = CreateRequest("/static/?pageSize=" + SmugglerOptions.BatchSize + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += attachmentInfo.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest("/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); } lastEtag = Etag.Parse(attachmentInfo.Last().Value <string>("Etag")); } }
public static JsonDocument GetHighestJsonDocumentByEtag(List <JsonDocument> past) { var highest = new ComparableByteArray(Guid.Empty); JsonDocument highestDoc = null; for (int i = past.Count - 1; i >= 0; i--) { Guid etag = past[i].Etag.Value; if (highest.CompareTo(etag) > 0) { continue; } highest = new ComparableByteArray(etag); highestDoc = past[i]; } return(highestDoc); }
private static JsonDocument GetHighestEtag(JsonDocument[] past) { var highest = new ComparableByteArray(Guid.Empty); JsonDocument highestDoc = null; for (int i = past.Length - 1; i >= 0; i--) { var etag = past[i].Etag.Value; if (highest.CompareTo(etag) > 0) { continue; } highest = new ComparableByteArray(etag); highestDoc = past[i]; } return(highestDoc); }
private async static Task <Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize) { Etag lastEtag = databaseOptions.StartAttachmentsEtag; int totalCount = 0; while (true) { var attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize); if (attachments.Length == 0) { var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += attachments.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount); foreach (var attachmentInformation in attachments) { if (databaseOptions.StripReplicationInformation) { attachmentInformation.Metadata = StripReplicationInformationFromMetadata(attachmentInformation.Metadata); } ShowProgress("Downloading attachment: {0}", attachmentInformation.Key); var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key); await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata); } lastEtag = Etag.Parse(attachments.Last().Etag); } }
private Guid ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Guid lastEtag) { int totalCount = 0; while (true) { var watch = Stopwatch.StartNew(); var documents = GetDocuments(lastEtag); watch.Stop(); if (documents.Length == 0) { var databaseStatistics = GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return(lastEtag); } var currentProcessingTime = watch.Elapsed; ModifyBatchSize(options, currentProcessingTime); var final = documents.Where(options.MatchFilters).ToList(); if (options.ShouldExcludeExpired) { final = documents.Where(options.ExcludeExpired).ToList(); } final.ForEach(item => item.WriteTo(jsonWriter)); totalCount += final.Count; ShowProgress("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount); lastEtag = new Guid(documents.Last().Value <RavenJObject>("@metadata").Value <string>("@etag")); } }
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { using (var documents = await GetDocuments(lastEtag)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { var document = documents.Current; if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); if (watch.ElapsedMilliseconds > 100) { errorcount++; } watch.Start(); } } var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return(lastEtag); } }
public void CleanupDocuments(Etag lastIndexedEtag) { var highest = new ComparableByteArray(lastIndexedEtag); foreach (var docToRemove in documentsToRemove) { if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false) continue; HashSet<Etag> _; documentsToRemove.TryRemove(docToRemove.Key, out _); } foreach (var updatedDocs in updatedDocuments) { if (updatedDocs.Value.All(etag => highest.CompareTo(etag) > 0) == false) continue; HashSet<Etag> _; updatedDocuments.TryRemove(updatedDocs.Key, out _); } JsonDocument result; while (prefetchingQueue.TryPeek(out result) && highest.CompareTo(result.Etag) >= 0) { prefetchingQueue.TryDequeue(out result); } }
public void CleanupDocumentsToRemove(Guid lastIndexedEtag) { var highest = new ComparableByteArray(lastIndexedEtag); foreach (var docToRemove in documentsToRemove) { if (docToRemove.Value.All(etag => highest.CompareTo(etag) > 0) == false) continue; HashSet<Guid> _; documentsToRemove.TryRemove(docToRemove.Key, out _); } JsonDocument result; while (inMemoryDocs.TryPeek(out result) && highest.CompareTo(result.Etag.Value) >= 0) { inMemoryDocs.TryDequeue(out result); } }
private static Guid GetLowestEtag(JsonDocument[] past) { var lowest = new ComparableByteArray(past[0].Etag.Value); for (int i = 1; i < past.Length; i++) { var etag = past[i].Etag.Value; if (lowest.CompareTo(etag) < 0) { continue; } lowest = new ComparableByteArray(etag); } return lowest.ToGuid(); }
private async static Task<Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize) { Etag lastEtag = databaseOptions.StartAttachmentsEtag; int totalCount = 0; while (true) { var attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize); if (attachments.Length == 0) { var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += attachments.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount); foreach (var attachmentInformation in attachments) { ShowProgress("Downloading attachment: {0}", attachmentInformation.Key); var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key); await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata); } lastEtag = Etag.Parse(attachments.Last().Etag); } }
private async Task <Etag> ExportDocuments(ISmugglerDatabaseOperations exportOperations, ISmugglerDatabaseOperations importOperations, SmugglerDatabaseOptions databaseOptions) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = databaseOptions.Limit - totalCount; if (maxRecords > 0) { var amountToFetchFromServer = Math.Min(databaseOptions.BatchSize, maxRecords); using (var documents = await exportOperations.GetDocuments(lastEtag, amountToFetchFromServer).ConfigureAwait(false)) { while (await documents.MoveNextAsync().ConfigureAwait(false)) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id"))); lastEtag = tempLastEtag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); await importOperations.PutDocument(document, (int)DocumentHelpers.GetRoughSize(document)).ConfigureAwait(false); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await exportOperations.GetStats().ConfigureAwait(false); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, amountToFetchFromServer); if (lastEtag.CompareTo(databaseStatistics.LastDocEtag) >= 0) { lastEtag = databaseStatistics.LastDocEtag; } ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { JsonDocument doc = exportOperations.GetDocument("Raven/Hilo/" + collectionName); if (doc != null) { doc.Metadata["@id"] = doc.Key; var jsonDoc = doc.ToJson(); AsyncHelpers.RunSync(() => importOperations.PutDocument(jsonDoc, (int)DocumentHelpers.GetRoughSize(jsonDoc))); totalCount++; } }); } }); await importOperations.PutDocument(null, -1).ConfigureAwait(false); // force flush ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
private void CleanupDocumentsToRemove(Guid lastIndexedEtag) { var highest = new ComparableByteArray(lastIndexedEtag); documentsToRemove.RemoveWhere(x => x.Etag == null || highest.CompareTo(x.Etag) <= 0); }
protected async Task<Etag> ExportDocuments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorsCount = 0; var reachedMaxEtag = false; Operations.ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = SmugglerOptions.Limit - totalCount; if (maxRecords > 0 && reachedMaxEtag == false) { using (var documents = await Operations.GetDocuments(src, lastEtag, Math.Min(SmugglerOptions.BatchSize, maxRecords))) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag")); Debug.Assert(!String.IsNullOrWhiteSpace(document.Value<RavenJObject>("@metadata").Value<string>("@id"))); if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0) { reachedMaxEtag = true; break; } lastEtag = tempLastEtag; if (!SmugglerOptions.MatchFilters(document)) continue; if (SmugglerOptions.ShouldExcludeExpired && SmugglerOptions.ExcludeExpired(document, now)) continue; document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { Operations.ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) errorsCount++; watch.Start(); } } if (hasDocs) continue; // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await Operations.GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); Operations.ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { Operations.ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); Operations.ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } // Load HiLo documents for selected collections SmugglerOptions.Filters.ForEach(filter => { if (filter.Path == "@metadata.Raven-Entity-Name") { filter.Values.ForEach(collectionName => { JsonDocument doc = Operations.GetDocument("Raven/Hilo/" + collectionName); if (doc != null) { doc.Metadata["@id"] = doc.Key; doc.ToJson().WriteTo(jsonWriter); totalCount++; } }); } }); Operations.ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); try { while (true) { try { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) continue; metadata = document.Value<RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; if (databaseOptions.StripReplicationInformation) document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); if (databaseOptions.ShouldDisableVersioningBundle) document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.Empty; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) return; doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
protected virtual async Task <Etag> ExportDocuments(RavenConnectionStringOptions src, SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorsCount = 0; var reachedMaxEtag = false; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0 && reachedMaxEtag == false) { using (var documents = await GetDocuments(src, lastEtag, Math.Min(options.BatchSize, maxRecords))) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; var tempLastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); Debug.Assert(!String.IsNullOrWhiteSpace(document.Value <RavenJObject>("@metadata").Value <string>("@id"))); if (maxEtag != null && tempLastEtag.CompareTo(maxEtag) > 0) { reachedMaxEtag = true; break; } lastEtag = tempLastEtag; if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) { errorsCount++; } watch.Start(); } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } // Load HiLo documents for selected collections options.Filters.ForEach(filter => { if (filter.Path == "@metadata.Raven-Entity-Name") { filter.Values.ForEach(collectionName => { JsonDocument doc = GetDocument("Raven/Hilo/" + collectionName); if (doc != null) { doc.Metadata["@id"] = doc.Key; doc.ToJson().WriteTo(jsonWriter); totalCount++; } }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; var lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); var isLastLoop = false; try { while (true) { try { var beforeCount = totalCount; if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); var res = await TransferStreamedDocuments(exportStore, databaseOptions, now, jintHelper, bulkInsertOperation, reportInterval, totalCount, lastEtag, lastReport); totalCount = res.Item1; lastEtag = res.Item2; lastReport = res.Item3; } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var res = await TransferDocumentsWithoutStreaming(exportStore, databaseOptions, exportBatchSize, operationMetadata, now, bulkInsertOperation, reportInterval, totalCount, lastEtag, lastReport); totalCount = res.Item1; lastEtag = res.Item2; lastReport = res.Item3; break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.Empty; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop) { if (totalCount == beforeCount) { isLastLoop = true; ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag); } else ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) return; doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0) { using (var documents = await GetDocuments(lastEtag, maxRecords)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag")); if (!options.MatchFilters(document)) continue; if (options.ShouldExcludeExpired && options.ExcludeExpired(document)) continue; document.WriteTo(jsonWriter); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) errorcount++; watch.Start(); } } if (hasDocs) continue; // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return lastEtag; } }
private IEnumerable <IndexingBatchForIndex> FilterIndexes(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); var filteredDocs = BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc => { var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index); return(filteredDoc == null ? new { Doc = doc, Json = (object)new FilteredDocument(doc) } : new { Doc = filteredDoc, Json = JsonToExpando.Convert(doc.ToJson()) }); }); Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count); var results = new IndexingBatchForIndex[indexesToWorkOn.Count]; var actions = new Action <IStorageActionsAccessor> [indexesToWorkOn.Count]; BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) => { var indexLastIndexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()); if (indexLastIndexEtag.CompareTo(lastIndexedEtag) >= 0) { return; } var indexName = indexToWorkOn.IndexName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if (viewGenerator == null) { return; // probably deleted } var batch = new IndexingBatch(); foreach (var item in filteredDocs) { if (prefetchingBehavior.FilterDocuments(item.Doc)) { continue; } // did we already indexed this document in this index? var etag = item.Doc.Etag; if (etag == null) { continue; } if (indexLastIndexEtag.CompareTo(new ComparableByteArray(etag.Value.ToByteArray())) >= 0) { continue; } // is the Raven-Entity-Name a match for the things the index executes on? if (viewGenerator.ForEntityNames.Count != 0 && viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value <string>(Constants.RavenEntityName)) == false) { continue; } batch.Add(item.Doc, item.Json, prefetchingBehavior.ShouldSkipDeleteFromIndex(item.Doc)); if (batch.DateTime == null) { batch.DateTime = item.Doc.LastModified; } else { batch.DateTime = batch.DateTime > item.Doc.LastModified ? item.Doc.LastModified : batch.DateTime; } } if (batch.Docs.Count == 0) { Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName, lastEtag, lastModified); // we use it this way to batch all the updates together actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified); return; } if (Log.IsDebugEnabled) { Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids)); } results[i] = new IndexingBatchForIndex { Batch = batch, IndexName = indexToWorkOn.IndexName, Index = indexToWorkOn.Index, LastIndexedEtag = indexToWorkOn.LastIndexedEtag }; }); transactionalStorage.Batch(actionsAccessor => { foreach (var action in actions) { if (action != null) { action(actionsAccessor); } } }); return(results.Where(x => x != null)); }
protected override async Task<Etag> ExportAttachments(RavenConnectionStringOptions src,JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { if (maxEtag != null) { throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag); } var totalCount = 0; while (true) { try { if (SmugglerOptions.Limit - totalCount <= 0) { ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize); RavenJArray attachmentInfo = null; var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value<string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest(src, "/static/" + item.Value<string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { {"Data", attachmentData}, {"Metadata", item.Value<RavenJObject>("Metadata")}, {"Key", item.Value<string>("Key")} } .WriteTo(jsonWriter); totalCount++; lastEtag = Etag.Parse(item.Value<string>("Etag")); } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
private async static Task <Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize) { Etag lastEtag = databaseOptions.StartAttachmentsEtag; int totalCount = 0; var retries = RetriesCount; while (true) { try { AttachmentInformation[] attachments; try { attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize); } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) { return(Etag.InvalidEtag); } if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("Failed to get attachments. {0} retries remaining. Message: {1}", retries, e.Message); continue; } if (attachments.Length == 0) { DatabaseStatistics databaseStatistics; try { databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) { return(Etag.Empty); } if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("Failed to get database statistics. Message: {0}", e.Message); continue; } var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } totalCount += attachments.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount); foreach (var attachmentInformation in attachments) { ShowProgress("Downloading attachment: {0}", attachmentInformation.Key); try { var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key); if (attachment == null) { continue; } if (databaseOptions.StripReplicationInformation) { attachment.Metadata = StripReplicationInformationFromMetadata(attachment.Metadata); } await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of an attachment {0} failed. Message: {1}", attachmentInformation.Key, e.Message); } } lastEtag = Etag.Parse(attachments.Last().Etag); } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
private void CleanupDocumentsToRemove(Guid lastIndexedEtag) { var highest = new ComparableByteArray(lastIndexedEtag); documentsToRemove.RemoveWhere(x => x.Etag == null || highest.CompareTo(x.Etag.Value) >= 0); }
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerOptions options, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = options.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); try { while (true) { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); document.Remove("@metadata"); bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = options.Timeout.Seconds; if (timeout < 30) { timeout = 30; } try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (RavenJObject document in documents) { var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } break; } catch (Exception e) { if (retries-- == 0) { throw; } exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return(lastEtag); } } finally { bulkInsertOperation.Dispose(); } }
private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { using (var documents = await GetDocuments(lastEtag)) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { var document = documents.Current; if (!options.MatchFilters(document)) continue; if (options.ShouldExcludeExpired && options.ExcludeExpired(document)) continue; document.WriteTo(jsonWriter); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag")); if (watch.ElapsedMilliseconds > 100) errorcount++; watch.Start(); } } var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } }
private static JsonDocument GetHighestEtag(JsonDocument[] past) { var highest = new ComparableByteArray(Guid.Empty); JsonDocument highestDoc = null; for (int i = past.Length - 1; i >= 0; i--) { var etag = past[i].Etag.Value; if (highest.CompareTo(etag) > 0) { continue; } highest = new ComparableByteArray(etag); highestDoc = past[i]; } return highestDoc; }
protected override async Task <Etag> ExportAttachments(RavenConnectionStringOptions src, JsonTextWriter jsonWriter, Etag lastEtag, Etag maxEtag) { if (maxEtag != null) { throw new ArgumentException("We don't support maxEtag in SmugglerApi", maxEtag); } var totalCount = 0; while (true) { try { if (SmugglerOptions.Limit - totalCount <= 0) { ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } var maxRecords = Math.Min(SmugglerOptions.Limit - totalCount, SmugglerOptions.BatchSize); RavenJArray attachmentInfo = null; var request = CreateRequest(src, "/static/?pageSize=" + maxRecords + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return(lastEtag); } ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value <string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest(src, "/static/" + item.Value <string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { { "Data", attachmentData }, { "Metadata", item.Value <RavenJObject>("Metadata") }, { "Key", item.Value <string>("Key") } } .WriteTo(jsonWriter); totalCount++; lastEtag = Etag.Parse(item.Value <string>("Etag")); } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
private IEnumerable<Tuple<IndexToWorkOn, IndexingBatch>> FilterIndexes(IList<IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); var documentRetriever = new DocumentRetriever(null, context.ReadTriggers); var filteredDocs = BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc => { var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index); return filteredDoc == null ? new { Doc = doc, Json = (object)new FilteredDocument(doc) } : new { Doc = filteredDoc, Json = JsonToExpando.Convert(doc.ToJson()) }; }); Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count); var results = new Tuple<IndexToWorkOn, IndexingBatch>[indexesToWorkOn.Count]; var actions = new Action<IStorageActionsAccessor>[indexesToWorkOn.Count]; BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) => { var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()); if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0) return; var indexName = indexToWorkOn.IndexName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if (viewGenerator == null) return; // probably deleted var batch = new IndexingBatch(); foreach (var item in filteredDocs) { if (FilterDocuments(item.Doc)) continue; // did we already indexed this document in this index? var etag = item.Doc.Etag; if (etag == null) continue; if (indexLastInedexEtag.CompareTo(new ComparableByteArray(etag.Value.ToByteArray())) >= 0) continue; // is the Raven-Entity-Name a match for the things the index executes on? if (viewGenerator.ForEntityNames.Count != 0 && viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value<string>(Constants.RavenEntityName)) == false) { continue; } batch.Add(item.Doc, item.Json); if (batch.DateTime == null) batch.DateTime = item.Doc.LastModified; else batch.DateTime = batch.DateTime > item.Doc.LastModified ? item.Doc.LastModified : batch.DateTime; } if (batch.Docs.Count == 0) { Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName, lastEtag, lastModified); // we use it this way to batch all the updates together actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified); return; } if (Log.IsDebugEnabled) { Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids)); } results[i] = Tuple.Create(indexToWorkOn, batch); }); transactionalStorage.Batch(actionsAccessor => { foreach (var action in actions) { if (action != null) action(actionsAccessor); } }); return results.Where(x => x != null); }
private IEnumerable<Tuple<IndexToWorkOn, IEnumerable<JsonDocument>>> FilterIndexes(IEnumerable<IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs) { var last = jsonDocs.Last(); Debug.Assert(last.Etag != null); Debug.Assert(last.LastModified != null); var lastEtag = last.Etag.Value; var lastModified = last.LastModified.Value; var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray()); Action<IStorageActionsAccessor> action = null; foreach (var indexToWorkOn in indexesToWorkOn) { var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray()); if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0) continue; var filteredDocs = jsonDocs.Where(doc => indexLastInedexEtag.CompareTo(new ComparableByteArray(doc.Etag.Value.ToByteArray())) < 0); var indexName = indexToWorkOn.IndexName; var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName); if(viewGenerator == null) continue; // probably deleted if (viewGenerator.ForEntityNames.Count != 0) // limit for the items that it care for { filteredDocs = filteredDocs.Where(x => viewGenerator.ForEntityNames.Contains(x.Metadata.Value<string>(Constants.RavenEntityName))); } List<JsonDocument> jsonDocuments = filteredDocs.ToList(); if(jsonDocuments.Count == 0) { // we use it this way to batch all the updates together action += accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified); continue; } yield return Tuple.Create<IndexToWorkOn, IEnumerable<JsonDocument>>(indexToWorkOn, jsonDocuments); } if (action != null) { transactionalStorage.Batch(action); } }
private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); try { while (true) { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); document.Remove("@metadata"); bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) timeout = 30; try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (RavenJObject document in documents) { var metadata = document.Value<RavenJObject>("@metadata"); var id = metadata.Value<string>("@id"); var etag = Etag.Parse(metadata.Value<string>("@etag")); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); if (!databaseOptions.MatchFilters(document)) continue; if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) continue; bulkInsertOperation.Store(document, metadata, id); totalCount++; if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } lastEtag = etag; } break; } catch (Exception e) { if (retries-- == 0) throw; exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } } finally { bulkInsertOperation.Dispose(); } }
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; var lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); var isLastLoop = false; try { while (true) { try { var beforeCount = totalCount; if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); var res = await TransferStreamedDocuments(exportStore, databaseOptions, now, jintHelper, bulkInsertOperation, reportInterval, totalCount, lastEtag, lastReport); totalCount = res.Item1; lastEtag = res.Item2; lastReport = res.Item3; } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) { timeout = 30; } try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var res = await TransferDocumentsWithoutStreaming(exportStore, databaseOptions, exportBatchSize, operationMetadata, now, bulkInsertOperation, reportInterval, totalCount, lastEtag, lastReport); totalCount = res.Item1; lastEtag = res.Item2; lastReport = res.Item3; break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) { return(Etag.Empty); } if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) <= 0 && !isLastLoop) { if (totalCount == beforeCount) { isLastLoop = true; ShowProgress("Got no new results , trying one more loop from: {0}", lastEtag); } else { ShowProgress("Finished streaming batch, but haven't reached an end (last reached etag = {0})", lastEtag); } continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) { return; } doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
public static JsonDocument GetHighestJsonDocumentByEtag(List<JsonDocument> past) { var highest = new ComparableByteArray(Etag.Empty); JsonDocument highestDoc = null; for (int i = past.Count - 1; i >= 0; i--) { Etag etag = past[i].Etag; if (highest.CompareTo(etag) > 0) { continue; } highest = new ComparableByteArray(etag); highestDoc = past[i]; } return highestDoc; }
private async Task <Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag) { var now = SystemTime.UtcNow; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); var errorcount = 0; ShowProgress("Exporting Documents"); while (true) { bool hasDocs = false; try { var maxRecords = options.Limit - totalCount; if (maxRecords > 0) { using (var documents = await GetDocuments(lastEtag, Math.Min(maxRecords, options.BatchSize))) { var watch = Stopwatch.StartNew(); while (await documents.MoveNextAsync()) { hasDocs = true; var document = documents.Current; lastEtag = Etag.Parse(document.Value <RavenJObject>("@metadata").Value <string>("@etag")); if (!options.MatchFilters(document)) { continue; } if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now)) { continue; } document.WriteTo(jsonWriter); totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } if (watch.ElapsedMilliseconds > 100) { errorcount++; } watch.Start(); } } if (hasDocs) { continue; } // The server can filter all the results. In this case, we need to try to go over with the next batch. // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop. // (This code provides support for legacy RavenDB version: 1.0) var databaseStatistics = await GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, maxRecords); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } } } catch (Exception e) { ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } }
protected override Guid ExportAttachments(JsonTextWriter jsonWriter, Guid lastEtag) { int totalCount = 0; while (true) { RavenJArray attachmentInfo = null; var request = CreateRequest("/static/?pageSize=" + smugglerOptions.BatchSize + "&etag=" + lastEtag); request.ExecuteRequest(reader => attachmentInfo = RavenJArray.Load(new JsonTextReader(reader))); if (attachmentInfo.Length == 0) { var databaseStatistics = GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += attachmentInfo.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachmentInfo.Length, totalCount); foreach (var item in attachmentInfo) { ShowProgress("Downloading attachment: {0}", item.Value<string>("Key")); byte[] attachmentData = null; var requestData = CreateRequest("/static/" + item.Value<string>("Key")); requestData.ExecuteRequest(reader => attachmentData = reader.ReadData()); new RavenJObject { {"Data", attachmentData}, {"Metadata", item.Value<RavenJObject>("Metadata")}, {"Key", item.Value<string>("Key")} } .WriteTo(jsonWriter); } lastEtag = new Guid(attachmentInfo.Last().Value<string>("Etag")); } }
private static async Task <Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize) { var now = SystemTime.UtcNow; string lastEtag = databaseOptions.StartDocsEtag; var totalCount = 0; var lastReport = SystemTime.UtcNow; var reportInterval = TimeSpan.FromSeconds(2); ShowProgress("Exporting Documents"); var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions { BatchSize = importBatchSize, OverwriteExisting = true, }); bulkInsertOperation.Report += text => ShowProgress(text); var jintHelper = new SmugglerJintHelper(); jintHelper.Initialize(databaseOptions); try { while (true) { try { if (exportStoreSupportedFeatures.IsDocsStreamingSupported) { ShowProgress("Streaming documents from " + lastEtag); using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag)) { while (await documentsEnumerator.MoveNextAsync()) { var document = documentsEnumerator.Current; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); if (!string.IsNullOrEmpty(databaseOptions.TransformScript)) { document = jintHelper.Transform(databaseOptions.TransformScript, document); if (document == null) { continue; } metadata = document.Value <RavenJObject>("@metadata"); } document.Remove("@metadata"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } } } else { int retries = RetriesCount; var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout; var timeout = databaseOptions.Timeout.Seconds; if (timeout < 30) { timeout = 30; } try { var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey); while (true) { try { ShowProgress("Get documents from " + lastEtag); var documents = await((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata); foreach (var jToken in documents) { var document = (RavenJObject)jToken; var metadata = document.Value <RavenJObject>("@metadata"); var id = metadata.Value <string>("@id"); var etag = Etag.Parse(metadata.Value <string>("@etag")); lastEtag = etag; if (!databaseOptions.MatchFilters(document)) { continue; } if (databaseOptions.ShouldExcludeExpired && databaseOptions.ExcludeExpired(document, now)) { continue; } if (databaseOptions.StripReplicationInformation) { document["@metadata"] = StripReplicationInformationFromMetadata(document["@metadata"] as RavenJObject); } if (databaseOptions.ShouldDisableVersioningBundle) { document["@metadata"] = SmugglerHelper.DisableVersioning(document["@metadata"] as RavenJObject); } document["@metadata"] = SmugglerHelper.HandleConflictDocuments(document["@metadata"] as RavenJObject); document.Remove("@metadata"); metadata.Remove("@id"); metadata.Remove("@etag"); try { bulkInsertOperation.Store(document, metadata, id); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } ShowProgress("IMPORT of a document {0} failed. Message: {1}", document, e.Message); } totalCount++; if (totalCount % 1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval) { ShowProgress("Exported {0} documents", totalCount); lastReport = SystemTime.UtcNow; } } break; } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) { return(Etag.Empty); } if (databaseOptions.IgnoreErrorsAndContinue == false) { throw; } exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); importStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2); ShowProgress("Error reading from database, remaining attempts {0}, will retry. Error: {1}", retries, e); } } } finally { exportStore.JsonRequestFactory.RequestTimeout = originalRequestTimeout; } } // In a case that we filter all the results, the formEtag hasn't updaed to the latest, // but we still need to continue until we finish all the docs. var databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag); continue; } // Load HiLo documents for selected collections databaseOptions.Filters.ForEach(filter => { if (string.Equals(filter.Path, "@metadata.Raven-Entity-Name", StringComparison.OrdinalIgnoreCase)) { filter.Values.ForEach(collectionName => { var doc = exportStore.DatabaseCommands.Get("Raven/Hilo/" + collectionName); if (doc == null) { return; } doc.Metadata["@id"] = doc.Key; bulkInsertOperation.Store(doc.DataAsJson, doc.Metadata, doc.Key); totalCount++; }); } }); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); return(lastEtag); } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } } finally { bulkInsertOperation.Dispose(); } }
private static async Task<Etag> ExportAttachments(DocumentStore exportStore, DocumentStore importStore, SmugglerDatabaseOptions databaseOptions, int exportBatchSize) { Etag lastEtag = databaseOptions.StartAttachmentsEtag; int totalCount = 0; var retries = RetriesCount; while (true) { try { AttachmentInformation[] attachments; try { attachments = await exportStore.AsyncDatabaseCommands.GetAttachmentsAsync(0, lastEtag, exportBatchSize); } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.InvalidEtag; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("Failed to get attachments. {0} retries remaining. Message: {1}", retries, e.Message); continue; } if (attachments.Length == 0) { DatabaseStatistics databaseStatistics; try { databaseStatistics = await exportStore.AsyncDatabaseCommands.GetStatisticsAsync(); } catch (Exception e) { if (retries-- == 0 && databaseOptions.IgnoreErrorsAndContinue) return Etag.Empty; if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("Failed to get database statistics. Message: {0}", e.Message); continue; } var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastAttachmentEtag) < 0) { lastEtag = EtagUtil.Increment(lastEtag, exportBatchSize); ShowProgress("Got no results but didn't get to the last attachment etag, trying from: {0}", lastEtag); continue; } ShowProgress("Done with reading attachments, total: {0}", totalCount); return lastEtag; } totalCount += attachments.Length; ShowProgress("Reading batch of {0,3} attachments, read so far: {1,10:#,#;;0}", attachments.Length, totalCount); foreach (var attachmentInformation in attachments) { ShowProgress("Downloading attachment: {0}", attachmentInformation.Key); try { var attachment = await exportStore.AsyncDatabaseCommands.GetAttachmentAsync(attachmentInformation.Key); if (attachment == null) continue; if (databaseOptions.StripReplicationInformation) attachment.Metadata = StripReplicationInformationFromMetadata(attachment.Metadata); await importStore.AsyncDatabaseCommands.PutAttachmentAsync(attachment.Key, null, attachment.Data(), attachment.Metadata); } catch (Exception e) { if (databaseOptions.IgnoreErrorsAndContinue == false) throw; ShowProgress("IMPORT of an attachment {0} failed. Message: {1}", attachmentInformation.Key, e.Message); } } lastEtag = Etag.Parse(attachments.Last().Etag); } catch (Exception e) { ShowProgress("Got Exception during smuggler between. Exception: {0}. ", e.Message); ShowProgress("Done with reading attachments, total: {0}", totalCount, lastEtag); throw new SmugglerExportException(e.Message, e) { LastEtag = lastEtag, }; } } }
private Guid ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Guid lastEtag) { int totalCount = 0; while (true) { var watch = Stopwatch.StartNew(); var documents = GetDocuments(lastEtag); watch.Stop(); if (documents.Length == 0) { var databaseStatistics = GetStats(); var lastEtagComparable = new ComparableByteArray(lastEtag); if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0) { lastEtag = Etag.Increment(lastEtag, smugglerOptions.BatchSize); ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}",lastEtag); continue; } ShowProgress("Done with reading documents, total: {0}", totalCount); return lastEtag; } var currentProcessingTime = watch.Elapsed; ModifyBatchSize(options, currentProcessingTime); var final = documents.Where(options.MatchFilters).ToList(); if (options.ShouldExcludeExpired) final = documents.Where(options.ExcludeExpired).ToList(); final.ForEach(item => item.WriteTo(jsonWriter)); totalCount += final.Count; ShowProgress("Reading batch of {0,3} documents, read so far: {1,10:#,#;;0}", documents.Length, totalCount); lastEtag = new Guid(documents.Last().Value<RavenJObject>("@metadata").Value<string>("@etag")); } }