예제 #1
0
        public ExportResult Export(DocumentsOperationContext context, Stream destinationStream, Action <IOperationProgress> onProgress = null)
        {
            var result   = new ExportResult();
            var progress = new IndeterminateProgress();

            using (var gZipStream = new GZipStream(destinationStream, CompressionMode.Compress, leaveOpen: true))
                using (var writer = new BlittableJsonTextWriter(context, gZipStream))
                {
                    writer.WriteStartObject();

                    writer.WritePropertyName(("BuildVersion"));
                    writer.WriteInteger(40000);

                    if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
                    {
                        progress.Progress = "Exporting Documents";
                        onProgress?.Invoke(progress);
                        writer.WriteComma();
                        writer.WritePropertyName(("Docs"));

                        IEnumerable <Document> documents = Options.CollectionsToExport.Count != 0 ?
                                                           _database.DocumentsStorage.GetDocumentsFrom(context, Options.CollectionsToExport, StartDocsEtag ?? 0, int.MaxValue) :
                                                           _database.DocumentsStorage.GetDocumentsFrom(context, StartDocsEtag ?? 0, 0, int.MaxValue);

                        writer.WriteStartArray();

                        PatchDocument patch        = null;
                        PatchRequest  patchRequest = null;
                        if (string.IsNullOrWhiteSpace(Options.TransformScript) == false)
                        {
                            patch        = new PatchDocument(context.DocumentDatabase);
                            patchRequest = new PatchRequest
                            {
                                Script = Options.TransformScript
                            };
                        }

                        bool first = true;
                        foreach (var document in documents)
                        {
                            if (document == null)
                            {
                                continue;
                            }

                            if (Options.IncludeExpired == false && document.Expired(_database.Time.GetUtcNow()))
                            {
                                continue;
                            }

                            var patchResult = patch?.Apply(context, document, patchRequest);
                            if (patchResult != null && patchResult.ModifiedDocument.Equals(document.Data) == false)
                            {
                                document.Data = patchResult.ModifiedDocument;
                            }

                            using (document.Data)
                            {
                                if (first == false)
                                {
                                    writer.WriteComma();
                                }
                                first = false;

                                document.EnsureMetadata();
                                context.Write(writer, document.Data);
                                result.LastDocsEtag = document.Etag;
                            }
                            result.ExportedDocuments++;
                        }
                        writer.WriteEndArray();
                    }

                    if (Options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
                    {
                        var versioningStorage = _database.BundleLoader.VersioningStorage;
                        if (versioningStorage != null)
                        {
                            writer.WriteComma();
                            writer.WritePropertyName("RevisionDocuments");
                            writer.WriteStartArray();
                            var first             = true;
                            var revisionDocuments = Options.RevisionDocumentsLimit.HasValue
                            ? versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0, Options.RevisionDocumentsLimit.Value)
                            : versioningStorage.GetRevisionsAfter(context, StartRevisionDocumentsEtag ?? 0);
                            foreach (var revisionDocument in revisionDocuments)
                            {
                                if (revisionDocument == null)
                                {
                                    continue;
                                }

                                using (revisionDocument.Data)
                                {
                                    if (first == false)
                                    {
                                        writer.WriteComma();
                                    }
                                    first = false;

                                    revisionDocument.EnsureMetadata();
                                    context.Write(writer, revisionDocument.Data);
                                    result.LastRevisionDocumentsEtag = revisionDocument.Etag;
                                }
                            }
                            writer.WriteEndArray();
                        }
                    }

                    if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Indexes))
                    {
                        progress.Progress = "Exporting Indexes";
                        onProgress?.Invoke(progress);
                        writer.WriteComma();
                        writer.WritePropertyName("Indexes");
                        writer.WriteStartArray();
                        var isFirst = true;
                        foreach (var index in _database.IndexStore.GetIndexes())
                        {
                            if (isFirst == false)
                            {
                                writer.WriteComma();
                            }
                            isFirst = false;
                            IndexProcessor.Export(writer, index, context, Options.RemoveAnalyzers);
                        }
                        writer.WriteEndArray();
                    }

                    if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Transformers))
                    {
                        progress.Progress = "Exporting Transformers";
                        onProgress?.Invoke(progress);
                        writer.WriteComma();
                        writer.WritePropertyName(("Transformers"));
                        writer.WriteStartArray();
                        var isFirst = true;
                        foreach (var transformer in _database.TransformerStore.GetTransformers())
                        {
                            if (isFirst == false)
                            {
                                writer.WriteComma();
                            }

                            isFirst = false;

                            TransformerProcessor.Export(writer, transformer, context);
                        }
                        writer.WriteEndArray();
                    }

                    if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
                    {
                        progress.Progress = "Exporting Identities";
                        onProgress?.Invoke(progress);
                        writer.WriteComma();
                        writer.WritePropertyName(("Identities"));
                        writer.WriteStartArray();
                        var identities = _database.DocumentsStorage.GetIdentities(context);
                        var first      = true;
                        foreach (var identity in identities)
                        {
                            if (first == false)
                            {
                                writer.WriteComma();
                            }
                            first = false;

                            writer.WriteStartObject();
                            writer.WritePropertyName(("Key"));
                            writer.WriteString((identity.Key));
                            writer.WriteComma();
                            writer.WritePropertyName(("Value"));
                            writer.WriteString((identity.Value.ToString()));
                            writer.WriteEndObject();
                        }
                        writer.WriteEndArray();
                    }

                    writer.WriteEndObject();
                    progress.Progress = $"Finish Exported database to {Options.FileName}. Exported {result.ExportedDocuments}";
                    onProgress?.Invoke(progress);
                }
            return(result);
        }
        public void ExecuteReplicationOnce()
        {
            _orderedReplicaItems.Clear();
            try
            {
                var sp      = Stopwatch.StartNew();
                var timeout = Debugger.IsAttached ? 60 * 1000 : 1000;
                var configurationContext = _parent._configurationContext;
                using (configurationContext.OpenReadTransaction())
                {
                    while (sp.ElapsedMilliseconds < timeout)
                    {
                        LastEtag = _parent._lastSentIndexOrTransformerEtag;

                        _parent.CancellationToken.ThrowIfCancellationRequested();

                        var indexAndTransformerMetadata = _parent._database.IndexMetadataPersistence.GetAfter(
                            configurationContext.Transaction.InnerTransaction,
                            configurationContext, LastEtag + 1, 0, 1024);

                        using (var stream = new MemoryStream())
                        {
                            foreach (var item in indexAndTransformerMetadata)
                            {
                                _parent.CancellationToken.ThrowIfCancellationRequested();
                                stream.Position = 0;
                                using (var writer = new BlittableJsonTextWriter(configurationContext, stream))
                                {
                                    switch (item.Type)
                                    {
                                    case IndexEntryType.Index:
                                        var index = _parent._database.IndexStore.GetIndex(item.Id);
                                        if (index == null)     //precaution
                                        {
                                            throw new InvalidDataException(
                                                      $"Index with name {item.Name} has metadata, but is not at the index store. This is not supposed to happen and is likely a bug.");
                                        }

                                        try
                                        {
                                            IndexProcessor.Export(writer, index, configurationContext, false);
                                        }
                                        catch (InvalidOperationException e)
                                        {
                                            if (_log.IsInfoEnabled)
                                            {
                                                _log.Info(
                                                    $"Failed to export index definition for replication. Index name = {item.Name}",
                                                    e);
                                            }
                                        }
                                        break;

                                    case IndexEntryType.Transformer:
                                        var transformer = _parent._database.TransformerStore.GetTransformer(item.Id);
                                        if (transformer == null)     //precaution
                                        {
                                            throw new InvalidDataException(
                                                      $"Transformer with name {item.Name} has metadata, but is not at the transformer store. This is not supposed to happen and is likely a bug.");
                                        }

                                        try
                                        {
                                            TransformerProcessor.Export(writer, transformer,
                                                                        configurationContext);
                                        }
                                        catch (InvalidOperationException e)
                                        {
                                            if (_log.IsInfoEnabled)
                                            {
                                                _log.Info(
                                                    $"Failed to export transformer definition for replication. Transformer name = {item.Name}",
                                                    e);
                                            }
                                        }
                                        break;

                                    default:
                                        throw new ArgumentOutOfRangeException(nameof(item),
                                                                              "Unexpected item type in index/transformer metadata. This is not supposed to happen.");
                                    }

                                    writer.Flush();

                                    stream.Position = 0;
                                    var newItem = new ReplicationBatchIndexItem
                                    {
                                        Name         = item.Name,
                                        ChangeVector = item.ChangeVector,
                                        Etag         = item.Etag,
                                        Type         = (int)item.Type,
                                        Definition   =
                                            configurationContext.ReadForMemory(stream,
                                                                               "Index/Transformer Replication - Reading definition into memory")
                                    };

                                    AddReplicationItemToBatch(newItem);
                                }
                            }
                        }

                        // if we are at the end, we are done
                        if (LastEtag <=
                            _parent._database.IndexMetadataPersistence.ReadLastEtag(
                                configurationContext.Transaction.InnerTransaction))
                        {
                            break;
                        }
                    }


                    if (_log.IsInfoEnabled)
                    {
                        _log.Info(
                            $"Found {_orderedReplicaItems.Count:#,#;;0} indexes/transformers to replicate to {_parent.Destination.Database} @ {_parent.Destination.Url} in {sp.ElapsedMilliseconds:#,#;;0} ms.");
                    }

                    _parent.CancellationToken.ThrowIfCancellationRequested();

                    try
                    {
                        using (_parent._documentsContext.OpenReadTransaction())
                            SendIndexTransformerBatch();
                    }
                    catch (Exception e)
                    {
                        if (_log.IsInfoEnabled)
                        {
                            _log.Info("Failed to send index/transformer replication batch", e);
                        }
                        throw;
                    }
                }
            }
            finally
            {
                //release memory at the end of the operation
                foreach (var item in _orderedReplicaItems)
                {
                    item.Value.Definition.Dispose();
                }

                _orderedReplicaItems.Clear();
            }
        }