コード例 #1
0
ファイル: RavenDB_16705.cs プロジェクト: ikvm/ravendb
            static void ModifyAttachment(AttachmentsStorage attachmentStorage, DocumentsOperationContext context, string documentId, string attachmentName, string contentType)
            {
                var attachment = attachmentStorage.GetAttachment(context, documentId, attachmentName, AttachmentType.Document, null);

                using (var stream = new MemoryStream(new byte[] { 1, 2, 3 }))
                {
                    attachmentStorage.PutAttachment(context, documentId, attachmentName, contentType, attachment.Base64Hash.ToString(), null, stream, updateDocument: false);
                }
            }
コード例 #2
0
        private bool ShouldFilterOutDeletion(RavenEtlItem item)
        {
            if (_script.HasDeleteDocumentsBehaviors)
            {
                var collection = item.Collection ?? item.CollectionFromMetadata;
                var documentId = item.DocumentId;

                if (item.IsAttachmentTombstone)
                {
                    documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId;

                    Debug.Assert(collection == null);

                    var document = Database.DocumentsStorage.Get(Context, documentId);

                    if (document == null)
                    {
                        return(true); // document was deleted, no need to send DELETE of attachment tombstone
                    }
                    collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name;
                }

                Debug.Assert(collection != null);

                if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) ||
                    _script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function))
                {
                    object[] parameters;

                    if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase))
                    {
                        parameters = new object[] { documentId, collection }
                    }
                    ;
                    else
                    {
                        parameters = new object[] { documentId }
                    };

                    using (var result = BehaviorsScript.Run(Context, Context, function, parameters))
                    {
                        if (result.BooleanValue == null || result.BooleanValue == false)
                        {
                            return(true);
                        }
                    }
                }
            }

            return(false);
        }
コード例 #3
0
        public override void Transform(RavenEtlItem item)
        {
            Current = item;

            if (item.IsDelete == false)
            {
                if (_script.Transformation != null)
                {
                    if (_script.LoadToCollections.Length > 1 || _script.IsLoadedToDefaultCollection(item, _script.LoadToCollections[0]) == false)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);
                    }

                    SingleRun.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
                }
                else
                {
                    _commands.Add(new PutCommandDataWithBlittableJson(item.DocumentId, null, item.Document.Data));

                    if ((item.Document.Flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
                    {
                        HandleDocumentAttachments(item);
                    }
                }
            }
            else
            {
                if (_script.Transformation != null)
                {
                    Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
                    ApplyDeleteCommands(item, OperationType.Delete);
                }
                else
                {
                    if (item.IsAttachmentTombstone == false)
                    {
                        _commands.Add(new DeleteCommandData(item.DocumentId, null));
                    }
                    else
                    {
                        var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                        _commands.Add(new DeleteAttachmentCommandData(doc, attachmentName, null));
                    }
                }
            }
        }
コード例 #4
0
        public static bool FilterAttachment(DocumentsOperationContext context, RavenEtlItem item)
        {
            var documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(context, item.AttachmentTombstoneId).DocId;
            var document   = context.DocumentDatabase.DocumentsStorage.Get(context, documentId);

            if (document == null)
            {
                return(true); // document could be deleted, no need to send DELETE of tombstone, we can filter it out
            }
            var collection = context.DocumentDatabase.DocumentsStorage.ExtractCollectionName(context, document.Data).Name;

            item.Collection = collection;
            return(false);
        }
コード例 #5
0
        public bool MoveNext()
        {
            Current = null;

            while (_tombstones.MoveNext())
            {
                var current = _tombstones.Current;
                if (current.Type == _tombstoneType)
                {
                    if (_fromCollections == null)
                    {
                        Current = current;
                        return(true);
                    }

                    var tombstoneCollection = (string)current.Collection;

                    if (string.IsNullOrEmpty(tombstoneCollection))
                    {
                        if (_tombstoneType == Tombstone.TombstoneType.Attachment)
                        {
                            var documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(_context, current.LowerId).DocId;
                            var document   = _context.DocumentDatabase.DocumentsStorage.Get(_context, documentId);

                            if (document != null) // document could be deleted, no need to send DELETE of tombstone, we can filter it out
                            {
                                tombstoneCollection = _context.DocumentDatabase.DocumentsStorage.ExtractCollectionName(_context, document.Data).Name;
                            }
                        }
                        else
                        {
                            ThrowUnexpectedNullCollectionTombstone(_tombstoneType);
                        }
                    }

                    if (_fromCollections.Contains(tombstoneCollection, StringComparer.OrdinalIgnoreCase))
                    {
                        Current = current;
                        return(true);
                    }
                }

                _stats.RecordChangeVector(current.ChangeVector);
                _stats.RecordLastFilteredOutEtag(current.Etag);
            }

            return(false);
        }
コード例 #6
0
ファイル: AttachmentHandler.cs プロジェクト: ikvm/ravendb
        public async Task Exists()
        {
            var hash = GetStringQueryString("hash");

            using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                using (context.OpenReadTransaction())
                    using (Slice.From(context.Allocator, hash, out var hashSlice))
                    {
                        var count = AttachmentsStorage.GetCountOfAttachmentsForHash(context, hashSlice);
                        await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                        {
                            writer.WriteStartObject();
                            writer.WritePropertyName("Hash");
                            writer.WriteString(hash);
                            writer.WriteComma();
                            writer.WritePropertyName("Count");
                            writer.WriteInteger(count);
                            writer.WriteEndObject();
                        }
                    }
        }
コード例 #7
0
        private void ValidateAttachments(BlittableJsonReaderArray attachments, DocumentsOperationContext context, string id)
        {
            if (attachments == null)
            {
                throw new InvalidOperationException($"Can not put document (id={id}) with '{Constants.Documents.Metadata.Attachments}': null");
            }

            foreach (BlittableJsonReaderObject attachment in attachments)
            {
                if (attachment.TryGet(nameof(AttachmentName.Hash), out string hash) == false || hash == null)
                {
                    throw new InvalidOperationException($"Can not put document (id={id}) because it contains an attachment without an hash property.");
                }
                using (Slice.From(context.Allocator, hash, out var hashSlice))
                {
                    if (AttachmentsStorage.GetCountOfAttachmentsForHash(context, hashSlice) < 1)
                    {
                        throw new InvalidOperationException($"Can not put document (id={id}) because it contains an attachment with hash={hash} but no such attachment is stored.");
                    }
                }
            }
        }
コード例 #8
0
        public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
                                               string expectedChangeVector,
                                               BlittableJsonReaderObject document,
                                               long?lastModifiedTicks = null,
                                               string changeVector    = null,
                                               DocumentFlags flags    = DocumentFlags.None,
                                               NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
        {
            if (context.Transaction == null)
            {
                ThrowRequiresTransaction();
                return(default(PutOperationResults)); // never hit
            }

#if DEBUG
            var documentDebugHash = document.DebugHash;
            document.BlittableValidation();
            BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
            AssertMetadataWasFiltered(document);
#endif

            var newEtag       = _documentsStorage.GenerateNextEtag();
            var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;

            id = BuildDocumentId(id, newEtag, out bool knownNewId);
            using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
            {
                var collectionName = _documentsStorage.ExtractCollectionName(context, document);
                var table          = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));

                var oldValue = default(TableValueReader);
                if (knownNewId == false)
                {
                    // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
                    DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);

                    table.ReadByKey(lowerId, out oldValue);
                }

                BlittableJsonReaderObject oldDoc = null;
                if (oldValue.Pointer == null)
                {
                    // expectedChangeVector being null means we don't care, and empty
                    // means that it must be new
                    if (string.IsNullOrEmpty(expectedChangeVector) == false)
                    {
                        ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
                    }
                }
                else
                {
                    // expectedChangeVector  has special meaning here
                    // null - means, don't care, don't check
                    // "" / empty - means, must be new
                    // anything else - must match exactly
                    if (expectedChangeVector != null)
                    {
                        var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
                        if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
                        {
                            ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
                        }
                    }

                    oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
                    var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
                    if (oldCollectionName != collectionName)
                    {
                        ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
                    }

                    var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);

                    if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
                        (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
                    {
                        if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
                        {
                            flags |= DocumentFlags.HasAttachments;
                        }
                    }
                }

                var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
                changeVector        = result.ChangeVector;
                nonPersistentFlags |= result.NonPersistentFlags;
                if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
                {
                    flags |= DocumentFlags.Resolved;
                }

                if (collectionName.IsHiLo == false &&
                    (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
                {
                    if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
                    {
#if DEBUG
                        if (document.DebugHash != documentDebugHash)
                        {
                            throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
                                                           "this is likely because you are trying to save a document that is already stored and was moved");
                        }
#endif
                        document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
                        documentDebugHash = document.DebugHash;
                        document.BlittableValidation();
                        BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
                        AssertMetadataWasFiltered(document);
                        AttachmentsStorage.AssertAttachments(document, flags);
#endif
                    }

                    if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
                        (flags.Contain(DocumentFlags.Resolved) ||
                         _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
                        ))
                    {
                        var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
                                                                                                                      ref flags, out RevisionsCollectionConfiguration configuration);
                        if (shouldVersion)
                        {
                            _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
                                                                                    changeVector, modifiedTicks, configuration, collectionName);
                        }
                    }
                }

                using (Slice.From(context.Allocator, changeVector, out var cv))
                    using (table.Allocate(out TableValueBuilder tvb))
                    {
                        tvb.Add(lowerId);
                        tvb.Add(Bits.SwapBytes(newEtag));
                        tvb.Add(idPtr);
                        tvb.Add(document.BasePointer, document.Size);
                        tvb.Add(cv.Content.Ptr, cv.Size);
                        tvb.Add(modifiedTicks);
                        tvb.Add((int)flags);
                        tvb.Add(context.GetTransactionMarker());

                        if (oldValue.Pointer == null)
                        {
                            table.Insert(tvb);
                        }
                        else
                        {
                            table.Update(oldValue.Id, tvb);
                        }
                    }

                if (collectionName.IsHiLo == false)
                {
                    _documentsStorage.ExpirationStorage.Put(context, lowerId, document);
                }

                context.LastDatabaseChangeVector = changeVector;
                _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
                _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);

                context.Transaction.AddAfterCommitNotification(new DocumentChange
                {
                    ChangeVector   = changeVector,
                    CollectionName = collectionName.Name,
                    Id             = id,
                    Type           = DocumentChangeTypes.Put,
                });

#if DEBUG
                if (document.DebugHash != documentDebugHash)
                {
                    throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
                                                   "this is likely because you are trying to save a document that is already stored and was moved");
                }
                document.BlittableValidation();
                BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
                AssertMetadataWasFiltered(document);
                AttachmentsStorage.AssertAttachments(document, flags);
#endif
                return(new PutOperationResults
                {
                    Etag = newEtag,
                    Id = id,
                    Collection = collectionName,
                    ChangeVector = changeVector,
                    Flags = flags,
                    LastModified = new DateTime(modifiedTicks)
                });
            }
        }
コード例 #9
0
        public override void Transform(RavenEtlItem item)
        {
            Current     = item;
            _currentRun = new RavenEtlScriptRun();

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);

                        DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();

                        if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
                        {
                            foreach (var counter in GetCountersFor(Current))
                            {
                                using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
                                {
                                    if (result.BooleanValue == true)
                                    {
                                        _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
                    }

                    break;

                case EtlItemType.Counter:
                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                    }

                    break;
                }
            }
            else
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        bool shouldDelete = true;

                        if (_script.HasDeleteDocumentsBehaviors && _script.TryGetDeleteDocumentBehaviorFunctionFor(item.Collection, out var function))
                        {
                            using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId }))
                            {
                                shouldDelete = result.BooleanValue == true;
                            }
                        }

                        if (shouldDelete)
                        {
                            ApplyDeleteCommands(item, OperationType.Delete);
                        }
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                    break;

                case EtlItemType.Counter:

                    var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);

                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.DeleteCounter(docId, counterName);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.DeleteCounter(docId, counterName);
                    }

                    break;
                }
            }

            _commands.AddRange(_currentRun.GetCommands());
        }
コード例 #10
0
        public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state)
        {
            Current = item;
            _currentRun ??= new RavenEtlScriptRun(stats);

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }))
                        {
                            ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted);

                            if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false)
                            {
                                break;
                            }

                            if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction))
                            {
                                var counterGroups = GetCounterGroupsFor(item);
                                if (counterGroups != null)
                                {
                                    AddCounters(item.DocumentId, counterGroups, counterFunction);
                                }
                            }

                            if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc))
                            {
                                if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state))
                                {
                                    var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc);
                                    if (timeSeriesReaders != null)
                                    {
                                        AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders);
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        var attachments          = GetAttachmentsFor(item);
                        var counterOperations    = GetCounterOperationsFor(item);
                        var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null;
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations);
                    }
                    break;

                case EtlItemType.CounterGroup:
                    string cFunction = null;
                    if (_script.HasTransformation)
                    {
                        if (_script.MayLoadToDefaultCollection(item) == false)
                        {
                            break;
                        }
                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false)
                        {
                            break;
                        }
                    }
                    AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction);
                    break;

                case EtlItemType.TimeSeries:
                    string tsFunction = null;
                    if (_script.HasTransformation)
                    {
                        if (_script.MayLoadToDefaultCollection(item) == false)
                        {
                            break;
                        }
                        if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false)
                        {
                            break;
                        }
                    }
                    HandleSingleTimeSeriesSegment(tsFunction, stats, state);
                    break;
                }
            }
            else
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (ShouldFilterOutDeletion(item))
                    {
                        break;
                    }
                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        ApplyDeleteCommands(item, OperationType.Delete, out _);
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                    break;

                case EtlItemType.TimeSeries:
                    string function = null;
                    if (_script.HasTransformation)
                    {
                        if (_script.MayLoadToDefaultCollection(item) == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false)
                        {
                            break;
                        }
                    }
                    HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function);
                    break;

                default:
                    throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}");
                }
            }
        }
コード例 #11
0
        public bool MoveNext()
        {
            Current = null;

            while (_tombstones.MoveNext())
            {
                var current = _tombstones.Current;

                if (_maxEtag != null)
                {
                    switch (_tombstoneType)
                    {
                    case Tombstone.TombstoneType.Counter:
                        if (PreventCountersIteratingTooFarEnumerator <ExtractedItem> .CanMoveNext(current.Etag, _maxEtag.Value) == false)
                        {
                            return(false);
                        }
                        break;

                    default:
                        ThrowMaxEtagLimitNotSupported(_tombstoneType);
                        break;
                    }
                }

                if (current.Type == _tombstoneType)
                {
                    if (_fromCollections == null)
                    {
                        Current = current;
                        return(true);
                    }

                    var tombstoneCollection = (string)current.Collection;

                    if (string.IsNullOrEmpty(tombstoneCollection))
                    {
                        if (_tombstoneType == Tombstone.TombstoneType.Attachment)
                        {
                            var documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(_context, current.LowerId).DocId;
                            var document   = _context.DocumentDatabase.DocumentsStorage.Get(_context, documentId);

                            if (document != null) // document could be deleted, no need to send DELETE of tombstone, we can filter it out
                            {
                                tombstoneCollection = _context.DocumentDatabase.DocumentsStorage.ExtractCollectionName(_context, document.Data).Name;
                            }
                        }
                        else
                        {
                            ThrowUnexpectedNullCollectionTombstone(_tombstoneType);
                        }
                    }

                    if (_fromCollections.Contains(tombstoneCollection, StringComparer.OrdinalIgnoreCase))
                    {
                        Current = current;
                        return(true);
                    }
                }

                var etlItemType = EtlItemType.None;

                switch (current.Type)
                {
                case Tombstone.TombstoneType.Document:
                case Tombstone.TombstoneType.Attachment:
                case Tombstone.TombstoneType.Revision:
                    etlItemType = EtlItemType.Document;
                    break;

                case Tombstone.TombstoneType.Counter:
                    etlItemType = EtlItemType.Counter;
                    break;

                default:
                    ThrowFilteringTombstonesOfTypeNotSupported(current.Type);
                    break;
                }

                _stats.RecordChangeVector(current.ChangeVector);

                _stats.RecordLastFilteredOutEtag(current.Etag, etlItemType);
            }

            return(false);
        }
コード例 #12
0
        public override void Transform(RavenEtlItem item)
        {
            Current     = item;
            _currentRun = new RavenEtlScriptRun();

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);

                        DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();

                        if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
                        {
                            var counters = GetCountersFor(Current);

                            if (counters != null && counters.Count > 0)
                            {
                                foreach (var counter in counters)
                                {
                                    using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
                                    {
                                        if (result.BooleanValue == true)
                                        {
                                            _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
                                        }
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
                    }

                    break;

                case EtlItemType.Counter:
                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                    }

                    break;
                }
            }
            else
            {
                switch (item.Type)
                {
                case EtlItemType.Document:

                    if (ShouldFilterOutDeletion())
                    {
                        break;
                    }

                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        ApplyDeleteCommands(item, OperationType.Delete);
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                    break;

                case EtlItemType.Counter:

                    var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);

                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.DeleteCounter(docId, counterName);
                            }
                        }
                    }
                    else
                    {
                        if (ShouldFilterOutDeletion())
                        {
                            break;
                        }

                        _currentRun.DeleteCounter(docId, counterName);
                    }

                    break;
                }

                bool ShouldFilterOutDeletion()
                {
                    if (_script.HasDeleteDocumentsBehaviors)
                    {
                        var collection = item.Collection ?? item.CollectionFromMetadata;
                        var documentId = item.DocumentId;

                        if (item.IsAttachmentTombstone)
                        {
                            documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId;

                            Debug.Assert(collection == null);

                            var document = Database.DocumentsStorage.Get(Context, documentId);

                            if (document == null)
                            {
                                return(true); // document was deleted, no need to send DELETE of attachment tombstone
                            }
                            collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name;
                        }
                        else if (item.Type == EtlItemType.Counter)
                        {
                            documentId = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId).DocId;
                        }

                        Debug.Assert(collection != null);

                        if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) ||
                            _script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function))
                        {
                            object[] parameters;

                            if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase))
                            {
                                parameters = new object[] { documentId, collection }
                            }
                            ;
                            else
                            {
                                parameters = new object[] { documentId }
                            };

                            using (var result = BehaviorsScript.Run(Context, Context, function, parameters))
                            {
                                if (result.BooleanValue == null || result.BooleanValue == false)
                                {
                                    return(true);
                                }
                            }
                        }
                    }

                    return(false);
                }
            }

            _commands.AddRange(_currentRun.GetCommands());
        }
コード例 #13
0
        public override void Transform(RavenEtlItem item, EtlStatsScope stats)
        {
            Current     = item;
            _currentRun = new RavenEtlScriptRun(stats);

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);

                        DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();

                        if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
                        {
                            var counterGroups = GetCounterGroupsFor(item);
                            if (counterGroups != null)
                            {
                                AddCounters(item.DocumentId, counterGroups, function);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item));
                    }

                    break;

                case EtlItemType.CounterGroup:
                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function);
                    }
                    else
                    {
                        AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument);
                    }

                    break;
                }
            }
            else
            {
                Debug.Assert(item.Type == EtlItemType.Document);

                if (ShouldFilterOutDeletion(item) == false)
                {
                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        ApplyDeleteCommands(item, OperationType.Delete);
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                }
            }

            _commands.AddRange(_currentRun.GetCommands());
        }