public ExportResult Export(DocumentsOperationContext context, Stream destinationStream, Action <IOperationProgress> onProgress = null) { var result = new ExportResult(); var progress = new IndeterminateProgress(); using (var gZipStream = new GZipStream(destinationStream, CompressionMode.Compress, leaveOpen: true)) using (var writer = new BlittableJsonTextWriter(context, gZipStream)) { writer.WriteStartObject(); writer.WritePropertyName(("BuildVersion")); writer.WriteInteger(40000); if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents)) { progress.Progress = "Exporting Documents"; onProgress?.Invoke(progress); writer.WriteComma(); writer.WritePropertyName(("Docs")); IEnumerable <Document> documents = Options.CollectionsToExport.Count != 0 ? _database.DocumentsStorage.GetDocumentsFrom(context, Options.CollectionsToExport, StartDocsEtag ?? 0, int.MaxValue) : _database.DocumentsStorage.GetDocumentsFrom(context, StartDocsEtag ?? 0, 0, int.MaxValue); writer.WriteStartArray(); PatchDocument patch = null; PatchRequest patchRequest = null; if (string.IsNullOrWhiteSpace(Options.TransformScript) == false) { patch = new PatchDocument(context.DocumentDatabase); patchRequest = new PatchRequest { Script = Options.TransformScript }; } bool first = true; foreach (var document in documents) { if (document == null) { continue; } if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow())) { continue; } var patchResult = patch?.Apply(context, document, patchRequest); if (patchResult != null && patchResult.ModifiedDocument.Equals(document.Data) == false) { document.Data = patchResult.ModifiedDocument; } using (document.Data) { if (first == false) { writer.WriteComma(); } first = false; document.EnsureMetadata(); context.Write(writer, document.Data); result.LastDocsEtag = document.Etag; } result.ExportedDocuments++; } writer.WriteEndArray(); } if (Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments)) { var versioningStorage = _database.BundleLoader.VersioningStorage; if (versioningStorage != null) { writer.WriteComma(); writer.WritePropertyName("RevisionDocuments"); writer.WriteStartArray(); var first = true; var revisionDocuments = Options.RevisionDocumentsLimit.HasValue ? versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0, Options.RevisionDocumentsLimit.Value) : versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0); foreach (var revisionDocument in revisionDocuments) { if (revisionDocument == null) { continue; } using (revisionDocument.Data) { if (first == false) { writer.WriteComma(); } first = false; revisionDocument.EnsureMetadata(); context.Write(writer, revisionDocument.Data); result.LastRevisionDocumentsEtag = revisionDocument.Etag; } } writer.WriteEndArray(); } } if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes)) { progress.Progress = "Exporting Indexes"; onProgress?.Invoke(progress); writer.WriteComma(); writer.WritePropertyName("Indexes"); writer.WriteStartArray(); var isFirst = true; foreach (var index in _database.IndexStore.GetIndexes()) { if (isFirst == false) { writer.WriteComma(); } isFirst = false; IndexProcessor.Export(writer, index, context, Options.RemoveAnalyzers); } writer.WriteEndArray(); } if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers)) { progress.Progress = "Exporting Transformers"; onProgress?.Invoke(progress); writer.WriteComma(); writer.WritePropertyName(("Transformers")); writer.WriteStartArray(); var isFirst = true; foreach (var transformer in _database.TransformerStore.GetTransformers()) { if (isFirst == false) { writer.WriteComma(); } isFirst = false; TransformerProcessor.Export(writer, transformer, context); } writer.WriteEndArray(); } if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities)) { progress.Progress = "Exporting Identities"; onProgress?.Invoke(progress); writer.WriteComma(); writer.WritePropertyName(("Identities")); writer.WriteStartArray(); var identities = _database.DocumentsStorage.GetIdentities(context); var first = true; foreach (var identity in identities) { if (first == false) { writer.WriteComma(); } first = false; writer.WriteStartObject(); writer.WritePropertyName(("Key")); writer.WriteString((identity.Key)); writer.WriteComma(); writer.WritePropertyName(("Value")); writer.WriteString((identity.Value.ToString())); writer.WriteEndObject(); } writer.WriteEndArray(); } writer.WriteEndObject(); progress.Progress = $"Finish Exported database to {Options.FileName}. Exported {result.ExportedDocuments}"; onProgress?.Invoke(progress); } return(result); }
public void ExecuteReplicationOnce() { _orderedReplicaItems.Clear(); try { var sp = Stopwatch.StartNew(); var timeout = Debugger.IsAttached ? 60 * 1000 : 1000; var configurationContext = _parent._configurationContext; using (configurationContext.OpenReadTransaction()) { while (sp.ElapsedMilliseconds < timeout) { LastEtag = _parent._lastSentIndexOrTransformerEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var indexAndTransformerMetadata = _parent._database.IndexMetadataPersistence.GetAfter( configurationContext.Transaction.InnerTransaction, configurationContext, LastEtag + 1, 0, 1024); using (var stream = new MemoryStream()) { foreach (var item in indexAndTransformerMetadata) { _parent.CancellationToken.ThrowIfCancellationRequested(); stream.Position = 0; using (var writer = new BlittableJsonTextWriter(configurationContext, stream)) { switch (item.Type) { case IndexEntryType.Index: var index = _parent._database.IndexStore.GetIndex(item.Id); if (index == null) //precaution { throw new InvalidDataException( $"Index with name {item.Name} has metadata, but is not at the index store. This is not supposed to happen and is likely a bug."); } try { IndexProcessor.Export(writer, index, configurationContext, false); } catch (InvalidOperationException e) { if (_log.IsInfoEnabled) { _log.Info( $"Failed to export index definition for replication. Index name = {item.Name}", e); } } break; case IndexEntryType.Transformer: var transformer = _parent._database.TransformerStore.GetTransformer(item.Id); if (transformer == null) //precaution { throw new InvalidDataException( $"Transformer with name {item.Name} has metadata, but is not at the transformer store. This is not supposed to happen and is likely a bug."); } try { TransformerProcessor.Export(writer, transformer, configurationContext); } catch (InvalidOperationException e) { if (_log.IsInfoEnabled) { _log.Info( $"Failed to export transformer definition for replication. Transformer name = {item.Name}", e); } } break; default: throw new ArgumentOutOfRangeException(nameof(item), "Unexpected item type in index/transformer metadata. This is not supposed to happen."); } writer.Flush(); stream.Position = 0; var newItem = new ReplicationBatchIndexItem { Name = item.Name, ChangeVector = item.ChangeVector, Etag = item.Etag, Type = (int)item.Type, Definition = configurationContext.ReadForMemory(stream, "Index/Transformer Replication - Reading definition into memory") }; AddReplicationItemToBatch(newItem); } } } // if we are at the end, we are done if (LastEtag <= _parent._database.IndexMetadataPersistence.ReadLastEtag( configurationContext.Transaction.InnerTransaction)) { break; } } if (_log.IsInfoEnabled) { _log.Info( $"Found {_orderedReplicaItems.Count:#,#;;0} indexes/transformers to replicate to {_parent.Destination.Database} @ {_parent.Destination.Url} in {sp.ElapsedMilliseconds:#,#;;0} ms."); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { using (_parent._documentsContext.OpenReadTransaction()) SendIndexTransformerBatch(); } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send index/transformer replication batch", e); } throw; } } } finally { //release memory at the end of the operation foreach (var item in _orderedReplicaItems) { item.Value.Definition.Dispose(); } _orderedReplicaItems.Clear(); } }