public override void Transform(ToSqlItem item, EtlStatsScope stats, EtlProcessState state) { _stats = stats; if (item.IsDelete == false) { Current = item; DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); } // ReSharper disable once ForCanBeConvertedToForeach for (int i = 0; i < _tablesForScript.Count; i++) { // delete all the rows that might already exist there var sqlTable = _tablesForScript[i]; if (sqlTable.InsertOnlyMode) { continue; } GetOrAdd(sqlTable.TableName).Deletes.Add(item); } }
public FilterCountersEnumerator(IEnumerator <CounterDetail> counters, EtlStatsScope stats, DocumentsStorage docsStorage, DocumentsOperationContext context) { _counters = counters; _stats = stats; _docsStorage = docsStorage; _context = context; }
public FilterCountersEnumerator(IEnumerator <CounterGroupDetail> counters, EtlStatsScope stats, DocumentsStorage docsStorage, DocumentsOperationContext context, long lastProcessedDocEtagInBatch) { _counters = counters; _stats = stats; _docsStorage = docsStorage; _context = context; _lastProcessedDocEtagInBatch = lastProcessedDocEtagInBatch; }
public FilterTombstonesEnumerator(IEnumerator <Tombstone> tombstones, EtlStatsScope stats, Tombstone.TombstoneType tombstoneType, DocumentsOperationContext context, List <string> fromCollections = null) { _tombstones = tombstones; _stats = stats; _tombstoneType = tombstoneType; _context = context; _fromCollections = fromCollections; }
private void HandleSingleTimeSeriesSegment(string loadBehaviorFunction, EtlStatsScope stats, EtlProcessState state) { var docId = Current.DocumentId; var segmentEntry = Current.TimeSeriesSegmentEntry; var doc = Database.DocumentsStorage.Get(Context, docId, DocumentFields.Default); if (doc == null) { //Through replication the Etl source database can have time-series without its document. //This is a rare situation and we will skip Etl this time-series and will mark the document so when it will be Etl we will send all its time-series with it (state.SkippedTimeSeriesDocs ??= new HashSet <string>()).Add(docId); return; } var timeSeriesEntries = segmentEntry.Segment.YieldAllValues(Context, segmentEntry.Start, false); if (loadBehaviorFunction != null && FilterSingleTimeSeriesSegmentByLoadBehaviorScript(ref timeSeriesEntries, docId, segmentEntry, loadBehaviorFunction)) { return; } if (doc.Etag > segmentEntry.Etag) { //There is a chance that the document didn't Etl yet so we push it with the time-series to be sure doc = Database.DocumentsStorage.Get(Context, docId); if (DocumentScript != null) { Current.Document = doc; DocumentScript.Run(Context, Context, "execute", new object[] { doc }).Dispose(); if (_currentRun.IsDocumentLoadedToSameCollection(docId) == false) { return; } } else { _currentRun.PutFullDocument(docId, doc.Data); } } var timeSeriesName = Database.DocumentsStorage.TimeSeriesStorage.GetTimeSeriesNameOriginalCasing(Context, docId, segmentEntry.Name); foreach (var entry in timeSeriesEntries) { _currentRun.AddTimeSeries(docId, timeSeriesName, entry); } }
public abstract void Transform(TExtracted item, EtlStatsScope stats);
public ExtractedItemsEnumerator(EtlStatsScope stats) { _extractionStats = stats.For(EtlOperations.Extract, start: false); }
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state) { Current = item; _currentRun ??= new RavenEtlScriptRun(stats); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document })) { ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted); if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction)) { var counterGroups = GetCounterGroupsFor(item); if (counterGroups != null) { AddCounters(item.DocumentId, counterGroups, counterFunction); } } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc)) { if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state)) { var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc); if (timeSeriesReaders != null) { AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders); } } } } } else { var attachments = GetAttachmentsFor(item); var counterOperations = GetCounterOperationsFor(item); var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null; _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations); } break; case EtlItemType.CounterGroup: string cFunction = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false) { break; } } AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction); break; case EtlItemType.TimeSeries: string tsFunction = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false) { break; } } HandleSingleTimeSeriesSegment(tsFunction, stats, state); break; } } else { switch (item.Type) { case EtlItemType.Document: if (ShouldFilterOutDeletion(item)) { break; } if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete, out _); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } break; case EtlItemType.TimeSeries: string function = null; if (_script.HasTransformation) { if (_script.MayLoadToDefaultCollection(item) == false) { break; } if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false) { break; } } HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function); break; default: throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}"); } } }
public abstract void Transform(TExtracted item, EtlStatsScope stats, EtlProcessState state);
public override void Transform(RavenEtlItem item, EtlStatsScope stats) { Current = item; _currentRun = new RavenEtlScriptRun(stats); if (item.IsDelete == false) { switch (item.Type) { case EtlItemType.Document: if (_script.HasTransformation) { // first, we need to delete docs prefixed by modified document ID to properly handle updates of // documents loaded to non default collections ApplyDeleteCommands(item, OperationType.Put); DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose(); if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function)) { var counterGroups = GetCounterGroupsFor(item); if (counterGroups != null) { AddCounters(item.DocumentId, counterGroups, function); } } } else { _currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item)); } break; case EtlItemType.CounterGroup: if (_script.HasTransformation) { if (_script.HasLoadCounterBehaviors == false) { break; } if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false) { break; } AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function); } else { AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument); } break; } } else { Debug.Assert(item.Type == EtlItemType.Document); if (ShouldFilterOutDeletion(item) == false) { if (_script.HasTransformation) { Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty"); ApplyDeleteCommands(item, OperationType.Delete); } else { if (item.IsAttachmentTombstone == false) { _currentRun.Delete(new DeleteCommandData(item.DocumentId, null)); } else { var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId); _currentRun.DeleteAttachment(doc, attachmentName); } } } } _commands.AddRange(_currentRun.GetCommands()); }
public FilterTombstonesEnumerator(IEnumerator <Tombstone> tombstones, EtlStatsScope stats) { _tombstones = tombstones; _stats = stats; }
public ExtractedItemsEnumerator(EtlStatsScope stats, EtlItemType itemType) { _itemType = itemType; _extractionStats = stats.For(EtlOperations.Extract, start: false); }