public override void Transform(RavenEtlItem item) { Current = item; _currentRun = new RavenEtlScriptRun(); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { foreach (var counter in GetCountersFor(Current)) { using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value); } } } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item)); } break; case EtlItemType.Counter: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } } } else { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } break; } } else { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); bool shouldDelete = true; if (_script.HasDeleteDocumentsBehaviors && _script.TryGetDeleteDocumentBehaviorFunctionFor(item.Collection, out var function)) { using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId })) { shouldDelete = result.BooleanValue == true; } } if (shouldDelete) { ApplyDeleteCommands(item, OperationType.Delete); } } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.Counter: var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId); if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName })) { if (result.BooleanValue == true) { _currentRun.DeleteCounter(docId, counterName); } } } else { _currentRun.DeleteCounter(docId, counterName); } break; } } _commands.AddRange(_currentRun.GetCommands()); }
public override void Transform(RavenEtlItem item) { Current = item; _currentRun = new RavenEtlScriptRun(); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { var counters = GetCountersFor(Current); if (counters != null && counters.Count > 0) { foreach (var counter in counters) { using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value); } } } } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item)); } break; case EtlItemType.Counter: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } } } else { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } break; } } else { switch (item.Type) { case EtlItemType.Document: if (ShouldFilterOutDeletion()) { break; } if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.Counter: var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId); if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName })) { if (result.BooleanValue == true) { _currentRun.DeleteCounter(docId, counterName); } } } else { if (ShouldFilterOutDeletion()) { break; } _currentRun.DeleteCounter(docId, counterName); } break; } bool ShouldFilterOutDeletion() { if (_script.HasDeleteDocumentsBehaviors) { var collection = item.Collection ?? item.CollectionFromMetadata; var documentId = item.DocumentId; if (item.IsAttachmentTombstone) { documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId; Debug.Assert(collection == null); var document = Database.DocumentsStorage.Get(Context, documentId); if (document == null) { return(true); // document was deleted, no need to send DELETE of attachment tombstone } collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name; } else if (item.Type == EtlItemType.Counter) { documentId = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId).DocId; } Debug.Assert(collection != null); if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) || _script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function)) { object[] parameters; if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase)) { parameters = new object[] { documentId, collection } } ; else { parameters = new object[] { documentId } }; using (var result = BehaviorsScript.Run(Context, Context, function, parameters)) { if (result.BooleanValue == null || result.BooleanValue == false) { return(true); } } } } return(false); } } _commands.AddRange(_currentRun.GetCommands()); }
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state) { Current = item; _currentRun ??= new RavenEtlScriptRun(stats); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document })) { ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted); if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction)) { var counterGroups = GetCounterGroupsFor(item); if (counterGroups != null) { AddCounters(item.DocumentId, counterGroups, counterFunction); } } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc)) { if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state)) { var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc); if (timeSeriesReaders != null) { AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders); } } } } } else { var attachments = GetAttachmentsFor(item); var counterOperations = GetCounterOperationsFor(item); var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null; _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations); } break; case EtlItemType.CounterGroup: string cFunction = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false) { break; } } AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction); break; case EtlItemType.TimeSeries: string tsFunction = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false) { break; } } HandleSingleTimeSeriesSegment(tsFunction, stats, state); break; } } else { switch (item.Type) { case EtlItemType.Document: if (ShouldFilterOutDeletion(item)) { break; } if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete, out _); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.TimeSeries: string function = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false) { break; } } HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function); break; default: throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}"); } } }
public override void Transform(RavenEtlItem item, EtlStatsScope stats) { Current = item; _currentRun = new RavenEtlScriptRun(stats); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { var counterGroups = GetCounterGroupsFor(item); if (counterGroups != null) { AddCounters(item.DocumentId, counterGroups, function); } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item)); } break; case EtlItemType.CounterGroup: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function); } else { AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument); } break; } } else { Debug.Assert(item.Type == EtlItemType.Document); if (ShouldFilterOutDeletion(item) == false) { if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } } } _commands.AddRange(_currentRun.GetCommands()); }