public override void Transform(RavenEtlItem item)
        {
            Current     = item;
            _currentRun = new RavenEtlScriptRun();

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);

                        DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();

                        if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
                        {
                            foreach (var counter in GetCountersFor(Current))
                            {
                                using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
                                {
                                    if (result.BooleanValue == true)
                                    {
                                        _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
                    }

                    break;

                case EtlItemType.Counter:
                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                    }

                    break;
                }
            }
            else
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        bool shouldDelete = true;

                        if (_script.HasDeleteDocumentsBehaviors && _script.TryGetDeleteDocumentBehaviorFunctionFor(item.Collection, out var function))
                        {
                            using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId }))
                            {
                                shouldDelete = result.BooleanValue == true;
                            }
                        }

                        if (shouldDelete)
                        {
                            ApplyDeleteCommands(item, OperationType.Delete);
                        }
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                    break;

                case EtlItemType.Counter:

                    var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);

                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.DeleteCounter(docId, counterName);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.DeleteCounter(docId, counterName);
                    }

                    break;
                }
            }

            _commands.AddRange(_currentRun.GetCommands());
        }
예제 #2
0
        public override void Transform(RavenEtlItem item)
        {
            Current     = item;
            _currentRun = new RavenEtlScriptRun();

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);

                        DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();

                        if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
                        {
                            var counters = GetCountersFor(Current);

                            if (counters != null && counters.Count > 0)
                            {
                                foreach (var counter in counters)
                                {
                                    using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
                                    {
                                        if (result.BooleanValue == true)
                                        {
                                            _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
                                        }
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
                    }

                    break;

                case EtlItemType.Counter:
                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
                    }

                    break;
                }
            }
            else
            {
                switch (item.Type)
                {
                case EtlItemType.Document:

                    if (ShouldFilterOutDeletion())
                    {
                        break;
                    }

                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        ApplyDeleteCommands(item, OperationType.Delete);
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                    break;

                case EtlItemType.Counter:

                    var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);

                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
                        {
                            if (result.BooleanValue == true)
                            {
                                _currentRun.DeleteCounter(docId, counterName);
                            }
                        }
                    }
                    else
                    {
                        if (ShouldFilterOutDeletion())
                        {
                            break;
                        }

                        _currentRun.DeleteCounter(docId, counterName);
                    }

                    break;
                }

                bool ShouldFilterOutDeletion()
                {
                    if (_script.HasDeleteDocumentsBehaviors)
                    {
                        var collection = item.Collection ?? item.CollectionFromMetadata;
                        var documentId = item.DocumentId;

                        if (item.IsAttachmentTombstone)
                        {
                            documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId;

                            Debug.Assert(collection == null);

                            var document = Database.DocumentsStorage.Get(Context, documentId);

                            if (document == null)
                            {
                                return(true); // document was deleted, no need to send DELETE of attachment tombstone
                            }
                            collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name;
                        }
                        else if (item.Type == EtlItemType.Counter)
                        {
                            documentId = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId).DocId;
                        }

                        Debug.Assert(collection != null);

                        if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) ||
                            _script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function))
                        {
                            object[] parameters;

                            if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase))
                            {
                                parameters = new object[] { documentId, collection }
                            }
                            ;
                            else
                            {
                                parameters = new object[] { documentId }
                            };

                            using (var result = BehaviorsScript.Run(Context, Context, function, parameters))
                            {
                                if (result.BooleanValue == null || result.BooleanValue == false)
                                {
                                    return(true);
                                }
                            }
                        }
                    }

                    return(false);
                }
            }

            _commands.AddRange(_currentRun.GetCommands());
        }
예제 #3
0
        public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state)
        {
            Current = item;
            _currentRun ??= new RavenEtlScriptRun(stats);

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }))
                        {
                            ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted);

                            if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false)
                            {
                                break;
                            }

                            if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction))
                            {
                                var counterGroups = GetCounterGroupsFor(item);
                                if (counterGroups != null)
                                {
                                    AddCounters(item.DocumentId, counterGroups, counterFunction);
                                }
                            }

                            if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc))
                            {
                                if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state))
                                {
                                    var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc);
                                    if (timeSeriesReaders != null)
                                    {
                                        AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders);
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        var attachments          = GetAttachmentsFor(item);
                        var counterOperations    = GetCounterOperationsFor(item);
                        var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null;
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations);
                    }
                    break;

                case EtlItemType.CounterGroup:
                    string cFunction = null;
                    if (_script.HasTransformation)
                    {
                        if (_script.MayLoadToDefaultCollection(item) == false)
                        {
                            break;
                        }
                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false)
                        {
                            break;
                        }
                    }
                    AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction);
                    break;

                case EtlItemType.TimeSeries:
                    string tsFunction = null;
                    if (_script.HasTransformation)
                    {
                        if (_script.MayLoadToDefaultCollection(item) == false)
                        {
                            break;
                        }
                        if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false)
                        {
                            break;
                        }
                    }
                    HandleSingleTimeSeriesSegment(tsFunction, stats, state);
                    break;
                }
            }
            else
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (ShouldFilterOutDeletion(item))
                    {
                        break;
                    }
                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        ApplyDeleteCommands(item, OperationType.Delete, out _);
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                    break;

                case EtlItemType.TimeSeries:
                    string function = null;
                    if (_script.HasTransformation)
                    {
                        if (_script.MayLoadToDefaultCollection(item) == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false)
                        {
                            break;
                        }
                    }
                    HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function);
                    break;

                default:
                    throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}");
                }
            }
        }
예제 #4
0
        public override void Transform(RavenEtlItem item, EtlStatsScope stats)
        {
            Current     = item;
            _currentRun = new RavenEtlScriptRun(stats);

            if (item.IsDelete == false)
            {
                switch (item.Type)
                {
                case EtlItemType.Document:
                    if (_script.HasTransformation)
                    {
                        // first, we need to delete docs prefixed by modified document ID to properly handle updates of
                        // documents loaded to non default collections

                        ApplyDeleteCommands(item, OperationType.Put);

                        DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();

                        if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
                        {
                            var counterGroups = GetCounterGroupsFor(item);
                            if (counterGroups != null)
                            {
                                AddCounters(item.DocumentId, counterGroups, function);
                            }
                        }
                    }
                    else
                    {
                        _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item));
                    }

                    break;

                case EtlItemType.CounterGroup:
                    if (_script.HasTransformation)
                    {
                        if (_script.HasLoadCounterBehaviors == false)
                        {
                            break;
                        }

                        if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
                        {
                            break;
                        }

                        AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function);
                    }
                    else
                    {
                        AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument);
                    }

                    break;
                }
            }
            else
            {
                Debug.Assert(item.Type == EtlItemType.Document);

                if (ShouldFilterOutDeletion(item) == false)
                {
                    if (_script.HasTransformation)
                    {
                        Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");

                        ApplyDeleteCommands(item, OperationType.Delete);
                    }
                    else
                    {
                        if (item.IsAttachmentTombstone == false)
                        {
                            _currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
                        }
                        else
                        {
                            var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);

                            _currentRun.DeleteAttachment(doc, attachmentName);
                        }
                    }
                }
            }

            _commands.AddRange(_currentRun.GetCommands());
        }