public override void Transform(ToSqlItem item, EtlStatsScope stats, EtlProcessState state)
{
_stats = stats;
if (item.IsDelete == false)
{
Current = item;
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
}
// ReSharper disable once ForCanBeConvertedToForeach
for (int i = 0; i < _tablesForScript.Count; i++)
{
// delete all the rows that might already exist there
var sqlTable = _tablesForScript[i];
if (sqlTable.InsertOnlyMode)
{
continue;
}
GetOrAdd(sqlTable.TableName).Deletes.Add(item);
}
}
public FilterCountersEnumerator(IEnumerator <CounterDetail> counters, EtlStatsScope stats, DocumentsStorage docsStorage, DocumentsOperationContext context)
{
_counters = counters;
_stats = stats;
_docsStorage = docsStorage;
_context = context;
}
public FilterCountersEnumerator(IEnumerator <CounterGroupDetail> counters, EtlStatsScope stats, DocumentsStorage docsStorage, DocumentsOperationContext context, long lastProcessedDocEtagInBatch)
{
_counters = counters;
_stats = stats;
_docsStorage = docsStorage;
_context = context;
_lastProcessedDocEtagInBatch = lastProcessedDocEtagInBatch;
}
public FilterTombstonesEnumerator(IEnumerator <Tombstone> tombstones, EtlStatsScope stats, Tombstone.TombstoneType tombstoneType, DocumentsOperationContext context,
List <string> fromCollections = null)
{
_tombstones = tombstones;
_stats = stats;
_tombstoneType = tombstoneType;
_context = context;
_fromCollections = fromCollections;
}
private void HandleSingleTimeSeriesSegment(string loadBehaviorFunction, EtlStatsScope stats, EtlProcessState state)
{
var docId = Current.DocumentId;
var segmentEntry = Current.TimeSeriesSegmentEntry;
var doc = Database.DocumentsStorage.Get(Context, docId, DocumentFields.Default);
if (doc == null)
{
//Through replication the Etl source database can have time-series without its document.
//This is a rare situation and we will skip Etl this time-series and will mark the document so when it will be Etl we will send all its time-series with it
(state.SkippedTimeSeriesDocs ??= new HashSet <string>()).Add(docId);
return;
}
var timeSeriesEntries = segmentEntry.Segment.YieldAllValues(Context, segmentEntry.Start, false);
if (loadBehaviorFunction != null && FilterSingleTimeSeriesSegmentByLoadBehaviorScript(ref timeSeriesEntries, docId, segmentEntry, loadBehaviorFunction))
{
return;
}
if (doc.Etag > segmentEntry.Etag)
{
//There is a chance that the document didn't Etl yet so we push it with the time-series to be sure
doc = Database.DocumentsStorage.Get(Context, docId);
if (DocumentScript != null)
{
Current.Document = doc;
DocumentScript.Run(Context, Context, "execute", new object[] { doc }).Dispose();
if (_currentRun.IsDocumentLoadedToSameCollection(docId) == false)
{
return;
}
}
else
{
_currentRun.PutFullDocument(docId, doc.Data);
}
}
var timeSeriesName = Database.DocumentsStorage.TimeSeriesStorage.GetTimeSeriesNameOriginalCasing(Context, docId, segmentEntry.Name);
foreach (var entry in timeSeriesEntries)
{
_currentRun.AddTimeSeries(docId, timeSeriesName, entry);
}
}
public abstract void Transform(TExtracted item, EtlStatsScope stats);
public ExtractedItemsEnumerator(EtlStatsScope stats)
{
_extractionStats = stats.For(EtlOperations.Extract, start: false);
}
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state)
{
Current = item;
_currentRun ??= new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }))
{
ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted);
if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, counterFunction);
}
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc))
{
if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state))
{
var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc);
if (timeSeriesReaders != null)
{
AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders);
}
}
}
}
}
else
{
var attachments = GetAttachmentsFor(item);
var counterOperations = GetCounterOperationsFor(item);
var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null;
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations);
}
break;
case EtlItemType.CounterGroup:
string cFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false)
{
break;
}
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction);
break;
case EtlItemType.TimeSeries:
string tsFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false)
{
break;
}
}
HandleSingleTimeSeriesSegment(tsFunction, stats, state);
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (ShouldFilterOutDeletion(item))
{
break;
}
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete, out _);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.TimeSeries:
string function = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false)
{
break;
}
}
HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function);
break;
default:
throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}");
}
}
}
public abstract void Transform(TExtracted item, EtlStatsScope stats, EtlProcessState state);
public override void Transform(RavenEtlItem item, EtlStatsScope stats)
{
Current = item;
_currentRun = new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, function);
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item));
}
break;
case EtlItemType.CounterGroup:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function);
}
else
{
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument);
}
break;
}
}
else
{
Debug.Assert(item.Type == EtlItemType.Document);
if (ShouldFilterOutDeletion(item) == false)
{
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
}
}
_commands.AddRange(_currentRun.GetCommands());
}
public FilterTombstonesEnumerator(IEnumerator <Tombstone> tombstones, EtlStatsScope stats)
{
_tombstones = tombstones;
_stats = stats;
}
public ExtractedItemsEnumerator(EtlStatsScope stats, EtlItemType itemType)
{
_itemType = itemType;
_extractionStats = stats.For(EtlOperations.Extract, start: false);
}
