public override void Transform(RavenEtlItem item)
{
Current = item;
_currentRun = new RavenEtlScriptRun();
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
foreach (var counter in GetCountersFor(Current))
{
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
}
}
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
}
break;
case EtlItemType.Counter:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
}
}
else
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
bool shouldDelete = true;
if (_script.HasDeleteDocumentsBehaviors && _script.TryGetDeleteDocumentBehaviorFunctionFor(item.Collection, out var function))
{
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId }))
{
shouldDelete = result.BooleanValue == true;
}
}
if (shouldDelete)
{
ApplyDeleteCommands(item, OperationType.Delete);
}
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.Counter:
var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
{
if (result.BooleanValue == true)
{
_currentRun.DeleteCounter(docId, counterName);
}
}
}
else
{
_currentRun.DeleteCounter(docId, counterName);
}
break;
}
}
_commands.AddRange(_currentRun.GetCommands());
}
public override void Transform(RavenEtlItem item)
{
Current = item;
_currentRun = new RavenEtlScriptRun();
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
var counters = GetCountersFor(Current);
if (counters != null && counters.Count > 0)
{
foreach (var counter in counters)
{
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
}
}
}
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
}
break;
case EtlItemType.Counter:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
}
}
else
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (ShouldFilterOutDeletion())
{
break;
}
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.Counter:
var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
{
if (result.BooleanValue == true)
{
_currentRun.DeleteCounter(docId, counterName);
}
}
}
else
{
if (ShouldFilterOutDeletion())
{
break;
}
_currentRun.DeleteCounter(docId, counterName);
}
break;
}
bool ShouldFilterOutDeletion()
{
if (_script.HasDeleteDocumentsBehaviors)
{
var collection = item.Collection ?? item.CollectionFromMetadata;
var documentId = item.DocumentId;
if (item.IsAttachmentTombstone)
{
documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId;
Debug.Assert(collection == null);
var document = Database.DocumentsStorage.Get(Context, documentId);
if (document == null)
{
return(true); // document was deleted, no need to send DELETE of attachment tombstone
}
collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name;
}
else if (item.Type == EtlItemType.Counter)
{
documentId = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId).DocId;
}
Debug.Assert(collection != null);
if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) ||
_script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function))
{
object[] parameters;
if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase))
{
parameters = new object[] { documentId, collection }
}
;
else
{
parameters = new object[] { documentId }
};
using (var result = BehaviorsScript.Run(Context, Context, function, parameters))
{
if (result.BooleanValue == null || result.BooleanValue == false)
{
return(true);
}
}
}
}
return(false);
}
}
_commands.AddRange(_currentRun.GetCommands());
}
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state)
{
Current = item;
_currentRun ??= new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }))
{
ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted);
if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, counterFunction);
}
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc))
{
if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state))
{
var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc);
if (timeSeriesReaders != null)
{
AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders);
}
}
}
}
}
else
{
var attachments = GetAttachmentsFor(item);
var counterOperations = GetCounterOperationsFor(item);
var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null;
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations);
}
break;
case EtlItemType.CounterGroup:
string cFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false)
{
break;
}
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction);
break;
case EtlItemType.TimeSeries:
string tsFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false)
{
break;
}
}
HandleSingleTimeSeriesSegment(tsFunction, stats, state);
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (ShouldFilterOutDeletion(item))
{
break;
}
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete, out _);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.TimeSeries:
string function = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false)
{
break;
}
}
HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function);
break;
default:
throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}");
}
}
}
public override void Transform(RavenEtlItem item, EtlStatsScope stats)
{
Current = item;
_currentRun = new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, function);
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item));
}
break;
case EtlItemType.CounterGroup:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function);
}
else
{
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument);
}
break;
}
}
else
{
Debug.Assert(item.Type == EtlItemType.Document);
if (ShouldFilterOutDeletion(item) == false)
{
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
}
}
_commands.AddRange(_currentRun.GetCommands());
}