public override void Transform(ToSqlItem item, EtlStatsScope stats, EtlProcessState state) { _stats = stats; if (item.IsDelete == false) { Current = item; DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); } // ReSharper disable once ForCanBeConvertedToForeach for (int i = 0; i < _tablesForScript.Count; i++) { // delete all the rows that might already exist there var sqlTable = _tablesForScript[i]; if (sqlTable.InsertOnlyMode) { continue; } GetOrAdd(sqlTable.TableName).Deletes.Add(item); } }
public override void Transform(ToOlapItem item, OlapEtlStatsScope stats, EtlProcessState state) { // Tombstones extraction is skipped by OLAP ETL. This must never happen Debug.Assert(item.IsDelete == false, $"Invalid item '{item.DocumentId}', OLAP ETL shouldn't handle tombstones"); _stats = stats; Current = item; DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); }
private void HandleSingleTimeSeriesSegment(string loadBehaviorFunction, EtlStatsScope stats, EtlProcessState state) { var docId = Current.DocumentId; var segmentEntry = Current.TimeSeriesSegmentEntry; var doc = Database.DocumentsStorage.Get(Context, docId, DocumentFields.Default); if (doc == null) { //Through replication the Etl source database can have time-series without its document. //This is a rare situation and we will skip Etl this time-series and will mark the document so when it will be Etl we will send all its time-series with it (state.SkippedTimeSeriesDocs ??= new HashSet <string>()).Add(docId); return; } var timeSeriesEntries = segmentEntry.Segment.YieldAllValues(Context, segmentEntry.Start, false); if (loadBehaviorFunction != null && FilterSingleTimeSeriesSegmentByLoadBehaviorScript(ref timeSeriesEntries, docId, segmentEntry, loadBehaviorFunction)) { return; } if (doc.Etag > segmentEntry.Etag) { //There is a chance that the document didn't Etl yet so we push it with the time-series to be sure doc = Database.DocumentsStorage.Get(Context, docId); if (DocumentScript != null) { Current.Document = doc; DocumentScript.Run(Context, Context, "execute", new object[] { doc }).Dispose(); if (_currentRun.IsDocumentLoadedToSameCollection(docId) == false) { return; } } else { _currentRun.PutFullDocument(docId, doc.Data); } } var timeSeriesName = Database.DocumentsStorage.TimeSeriesStorage.GetTimeSeriesNameOriginalCasing(Context, docId, segmentEntry.Name); foreach (var entry in timeSeriesEntries) { _currentRun.AddTimeSeries(docId, timeSeriesName, entry); } }
public override void Transform(ElasticSearchItem item, EtlStatsScope stats, EtlProcessState state) { if (item.IsDelete == false) { Current = item; DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); } for (int i = 0; i < _indexesForScript.Count; i++) { // delete all the rows that might already exist there var elasticIndex = _indexesForScript[i]; GetOrAdd(elasticIndex.IndexName).Deletes.Add(item); } }
public override void Transform(RavenEtlItem item) { Current = item; _currentRun = new RavenEtlScriptRun(); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { foreach (var counter in GetCountersFor(Current)) { using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value); } } } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item)); } break; case EtlItemType.Counter: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } } } else { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } break; } } else { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); bool shouldDelete = true; if (_script.HasDeleteDocumentsBehaviors && _script.TryGetDeleteDocumentBehaviorFunctionFor(item.Collection, out var function)) { using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId })) { shouldDelete = result.BooleanValue == true; } } if (shouldDelete) { ApplyDeleteCommands(item, OperationType.Delete); } } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.Counter: var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId); if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName })) { if (result.BooleanValue == true) { _currentRun.DeleteCounter(docId, counterName); } } } else { _currentRun.DeleteCounter(docId, counterName); } break; } } _commands.AddRange(_currentRun.GetCommands()); }
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state) { Current = item; _currentRun ??= new RavenEtlScriptRun(stats); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document })) { ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted); if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction)) { var counterGroups = GetCounterGroupsFor(item); if (counterGroups != null) { AddCounters(item.DocumentId, counterGroups, counterFunction); } } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc)) { if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state)) { var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc); if (timeSeriesReaders != null) { AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders); } } } } } else { var attachments = GetAttachmentsFor(item); var counterOperations = GetCounterOperationsFor(item); var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null; _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations); } break; case EtlItemType.CounterGroup: string cFunction = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false) { break; } } AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction); break; case EtlItemType.TimeSeries: string tsFunction = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false) { break; } } HandleSingleTimeSeriesSegment(tsFunction, stats, state); break; } } else { switch (item.Type) { case EtlItemType.Document: if (ShouldFilterOutDeletion(item)) { break; } if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete, out _); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.TimeSeries: string function = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false) { break; } } HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function); break; default: throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}"); } } }
public override void Transform(RavenEtlItem item) { Current = item; _currentRun = new RavenEtlScriptRun(); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { var counters = GetCountersFor(Current); if (counters != null && counters.Count > 0) { foreach (var counter in counters) { using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value); } } } } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item)); } break; case EtlItemType.Counter: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName })) { if (result.BooleanValue == true) { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } } } else { _currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue); } break; } } else { switch (item.Type) { case EtlItemType.Document: if (ShouldFilterOutDeletion()) { break; } if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.Counter: var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId); if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName })) { if (result.BooleanValue == true) { _currentRun.DeleteCounter(docId, counterName); } } } else { if (ShouldFilterOutDeletion()) { break; } _currentRun.DeleteCounter(docId, counterName); } break; } bool ShouldFilterOutDeletion() { if (_script.HasDeleteDocumentsBehaviors) { var collection = item.Collection ?? item.CollectionFromMetadata; var documentId = item.DocumentId; if (item.IsAttachmentTombstone) { documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId; Debug.Assert(collection == null); var document = Database.DocumentsStorage.Get(Context, documentId); if (document == null) { return(true); // document was deleted, no need to send DELETE of attachment tombstone } collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name; } else if (item.Type == EtlItemType.Counter) { documentId = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId).DocId; } Debug.Assert(collection != null); if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) || _script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function)) { object[] parameters; if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase)) { parameters = new object[] { documentId, collection } } ; else { parameters = new object[] { documentId } }; using (var result = BehaviorsScript.Run(Context, Context, function, parameters)) { if (result.BooleanValue == null || result.BooleanValue == false) { return(true); } } } } return(false); } } _commands.AddRange(_currentRun.GetCommands()); }
public override void Transform(RavenEtlItem item, EtlStatsScope stats) { Current = item; _currentRun = new RavenEtlScriptRun(stats); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { var counterGroups = GetCounterGroupsFor(item); if (counterGroups != null) { AddCounters(item.DocumentId, counterGroups, function); } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item)); } break; case EtlItemType.CounterGroup: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function); } else { AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument); } break; } } else { Debug.Assert(item.Type == EtlItemType.Document); if (ShouldFilterOutDeletion(item) == false) { if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } } } _commands.AddRange(_currentRun.GetCommands()); }