static void ModifyAttachment(AttachmentsStorage attachmentStorage, DocumentsOperationContext context, string documentId, string attachmentName, string contentType)
{
var attachment = attachmentStorage.GetAttachment(context, documentId, attachmentName, AttachmentType.Document, null);
using (var stream = new MemoryStream(new byte[] { 1, 2, 3 }))
{
attachmentStorage.PutAttachment(context, documentId, attachmentName, contentType, attachment.Base64Hash.ToString(), null, stream, updateDocument: false);
}
}
private bool ShouldFilterOutDeletion(RavenEtlItem item)
{
if (_script.HasDeleteDocumentsBehaviors)
{
var collection = item.Collection ?? item.CollectionFromMetadata;
var documentId = item.DocumentId;
if (item.IsAttachmentTombstone)
{
documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId;
Debug.Assert(collection == null);
var document = Database.DocumentsStorage.Get(Context, documentId);
if (document == null)
{
return(true); // document was deleted, no need to send DELETE of attachment tombstone
}
collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name;
}
Debug.Assert(collection != null);
if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) ||
_script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function))
{
object[] parameters;
if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase))
{
parameters = new object[] { documentId, collection }
}
;
else
{
parameters = new object[] { documentId }
};
using (var result = BehaviorsScript.Run(Context, Context, function, parameters))
{
if (result.BooleanValue == null || result.BooleanValue == false)
{
return(true);
}
}
}
}
return(false);
}
public override void Transform(RavenEtlItem item)
{
Current = item;
if (item.IsDelete == false)
{
if (_script.Transformation != null)
{
if (_script.LoadToCollections.Length > 1 || _script.IsLoadedToDefaultCollection(item, _script.LoadToCollections[0]) == false)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
}
SingleRun.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
}
else
{
_commands.Add(new PutCommandDataWithBlittableJson(item.DocumentId, null, item.Document.Data));
if ((item.Document.Flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
HandleDocumentAttachments(item);
}
}
}
else
{
if (_script.Transformation != null)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_commands.Add(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_commands.Add(new DeleteAttachmentCommandData(doc, attachmentName, null));
}
}
}
}
public static bool FilterAttachment(DocumentsOperationContext context, RavenEtlItem item)
{
var documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(context, item.AttachmentTombstoneId).DocId;
var document = context.DocumentDatabase.DocumentsStorage.Get(context, documentId);
if (document == null)
{
return(true); // document could be deleted, no need to send DELETE of tombstone, we can filter it out
}
var collection = context.DocumentDatabase.DocumentsStorage.ExtractCollectionName(context, document.Data).Name;
item.Collection = collection;
return(false);
}
public bool MoveNext()
{
Current = null;
while (_tombstones.MoveNext())
{
var current = _tombstones.Current;
if (current.Type == _tombstoneType)
{
if (_fromCollections == null)
{
Current = current;
return(true);
}
var tombstoneCollection = (string)current.Collection;
if (string.IsNullOrEmpty(tombstoneCollection))
{
if (_tombstoneType == Tombstone.TombstoneType.Attachment)
{
var documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(_context, current.LowerId).DocId;
var document = _context.DocumentDatabase.DocumentsStorage.Get(_context, documentId);
if (document != null) // document could be deleted, no need to send DELETE of tombstone, we can filter it out
{
tombstoneCollection = _context.DocumentDatabase.DocumentsStorage.ExtractCollectionName(_context, document.Data).Name;
}
}
else
{
ThrowUnexpectedNullCollectionTombstone(_tombstoneType);
}
}
if (_fromCollections.Contains(tombstoneCollection, StringComparer.OrdinalIgnoreCase))
{
Current = current;
return(true);
}
}
_stats.RecordChangeVector(current.ChangeVector);
_stats.RecordLastFilteredOutEtag(current.Etag);
}
return(false);
}
public async Task Exists()
{
var hash = GetStringQueryString("hash");
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
using (Slice.From(context.Allocator, hash, out var hashSlice))
{
var count = AttachmentsStorage.GetCountOfAttachmentsForHash(context, hashSlice);
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Hash");
writer.WriteString(hash);
writer.WriteComma();
writer.WritePropertyName("Count");
writer.WriteInteger(count);
writer.WriteEndObject();
}
}
}
private void ValidateAttachments(BlittableJsonReaderArray attachments, DocumentsOperationContext context, string id)
{
if (attachments == null)
{
throw new InvalidOperationException($"Can not put document (id={id}) with '{Constants.Documents.Metadata.Attachments}': null");
}
foreach (BlittableJsonReaderObject attachment in attachments)
{
if (attachment.TryGet(nameof(AttachmentName.Hash), out string hash) == false || hash == null)
{
throw new InvalidOperationException($"Can not put document (id={id}) because it contains an attachment without an hash property.");
}
using (Slice.From(context.Allocator, hash, out var hashSlice))
{
if (AttachmentsStorage.GetCountOfAttachmentsForHash(context, hashSlice) < 1)
{
throw new InvalidOperationException($"Can not put document (id={id}) because it contains an attachment with hash={hash} but no such attachment is stored.");
}
}
}
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care, and empty
// means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}
public override void Transform(RavenEtlItem item)
{
Current = item;
_currentRun = new RavenEtlScriptRun();
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
foreach (var counter in GetCountersFor(Current))
{
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
}
}
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
}
break;
case EtlItemType.Counter:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
}
}
else
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
bool shouldDelete = true;
if (_script.HasDeleteDocumentsBehaviors && _script.TryGetDeleteDocumentBehaviorFunctionFor(item.Collection, out var function))
{
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId }))
{
shouldDelete = result.BooleanValue == true;
}
}
if (shouldDelete)
{
ApplyDeleteCommands(item, OperationType.Delete);
}
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.Counter:
var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
{
if (result.BooleanValue == true)
{
_currentRun.DeleteCounter(docId, counterName);
}
}
}
else
{
_currentRun.DeleteCounter(docId, counterName);
}
break;
}
}
_commands.AddRange(_currentRun.GetCommands());
}
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state)
{
Current = item;
_currentRun ??= new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }))
{
ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted);
if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, counterFunction);
}
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc))
{
if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state))
{
var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc);
if (timeSeriesReaders != null)
{
AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders);
}
}
}
}
}
else
{
var attachments = GetAttachmentsFor(item);
var counterOperations = GetCounterOperationsFor(item);
var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null;
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations);
}
break;
case EtlItemType.CounterGroup:
string cFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false)
{
break;
}
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction);
break;
case EtlItemType.TimeSeries:
string tsFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false)
{
break;
}
}
HandleSingleTimeSeriesSegment(tsFunction, stats, state);
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (ShouldFilterOutDeletion(item))
{
break;
}
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete, out _);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.TimeSeries:
string function = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false)
{
break;
}
}
HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function);
break;
default:
throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}");
}
}
}
public bool MoveNext()
{
Current = null;
while (_tombstones.MoveNext())
{
var current = _tombstones.Current;
if (_maxEtag != null)
{
switch (_tombstoneType)
{
case Tombstone.TombstoneType.Counter:
if (PreventCountersIteratingTooFarEnumerator <ExtractedItem> .CanMoveNext(current.Etag, _maxEtag.Value) == false)
{
return(false);
}
break;
default:
ThrowMaxEtagLimitNotSupported(_tombstoneType);
break;
}
}
if (current.Type == _tombstoneType)
{
if (_fromCollections == null)
{
Current = current;
return(true);
}
var tombstoneCollection = (string)current.Collection;
if (string.IsNullOrEmpty(tombstoneCollection))
{
if (_tombstoneType == Tombstone.TombstoneType.Attachment)
{
var documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(_context, current.LowerId).DocId;
var document = _context.DocumentDatabase.DocumentsStorage.Get(_context, documentId);
if (document != null) // document could be deleted, no need to send DELETE of tombstone, we can filter it out
{
tombstoneCollection = _context.DocumentDatabase.DocumentsStorage.ExtractCollectionName(_context, document.Data).Name;
}
}
else
{
ThrowUnexpectedNullCollectionTombstone(_tombstoneType);
}
}
if (_fromCollections.Contains(tombstoneCollection, StringComparer.OrdinalIgnoreCase))
{
Current = current;
return(true);
}
}
var etlItemType = EtlItemType.None;
switch (current.Type)
{
case Tombstone.TombstoneType.Document:
case Tombstone.TombstoneType.Attachment:
case Tombstone.TombstoneType.Revision:
etlItemType = EtlItemType.Document;
break;
case Tombstone.TombstoneType.Counter:
etlItemType = EtlItemType.Counter;
break;
default:
ThrowFilteringTombstonesOfTypeNotSupported(current.Type);
break;
}
_stats.RecordChangeVector(current.ChangeVector);
_stats.RecordLastFilteredOutEtag(current.Etag, etlItemType);
}
return(false);
}
public override void Transform(RavenEtlItem item)
{
Current = item;
_currentRun = new RavenEtlScriptRun();
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
var counters = GetCountersFor(Current);
if (counters != null && counters.Count > 0)
{
foreach (var counter in counters)
{
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, counter.Name }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, counter.Name, counter.Value);
}
}
}
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCountersFor(item));
}
break;
case EtlItemType.Counter:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { item.DocumentId, item.CounterName }))
{
if (result.BooleanValue == true)
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
}
}
else
{
_currentRun.AddCounter(item.DocumentId, item.CounterName, item.CounterValue);
}
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (ShouldFilterOutDeletion())
{
break;
}
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.Counter:
var(docId, counterName) = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId);
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
using (var result = BehaviorsScript.Run(Context, Context, function, new object[] { docId, counterName }))
{
if (result.BooleanValue == true)
{
_currentRun.DeleteCounter(docId, counterName);
}
}
}
else
{
if (ShouldFilterOutDeletion())
{
break;
}
_currentRun.DeleteCounter(docId, counterName);
}
break;
}
bool ShouldFilterOutDeletion()
{
if (_script.HasDeleteDocumentsBehaviors)
{
var collection = item.Collection ?? item.CollectionFromMetadata;
var documentId = item.DocumentId;
if (item.IsAttachmentTombstone)
{
documentId = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId).DocId;
Debug.Assert(collection == null);
var document = Database.DocumentsStorage.Get(Context, documentId);
if (document == null)
{
return(true); // document was deleted, no need to send DELETE of attachment tombstone
}
collection = Database.DocumentsStorage.ExtractCollectionName(Context, document.Data).Name;
}
else if (item.Type == EtlItemType.Counter)
{
documentId = CountersStorage.ExtractDocIdAndCounterNameFromTombstone(Context, item.CounterTombstoneId).DocId;
}
Debug.Assert(collection != null);
if (_script.TryGetDeleteDocumentBehaviorFunctionFor(collection, out var function) ||
_script.TryGetDeleteDocumentBehaviorFunctionFor(Transformation.GenericDeleteDocumentsBehaviorFunctionKey, out function))
{
object[] parameters;
if (Transformation.GenericDeleteDocumentsBehaviorFunctionName.Equals(function, StringComparison.OrdinalIgnoreCase))
{
parameters = new object[] { documentId, collection }
}
;
else
{
parameters = new object[] { documentId }
};
using (var result = BehaviorsScript.Run(Context, Context, function, parameters))
{
if (result.BooleanValue == null || result.BooleanValue == false)
{
return(true);
}
}
}
}
return(false);
}
}
_commands.AddRange(_currentRun.GetCommands());
}
public override void Transform(RavenEtlItem item, EtlStatsScope stats)
{
Current = item;
_currentRun = new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
// first, we need to delete docs prefixed by modified document ID to properly handle updates of
// documents loaded to non default collections
ApplyDeleteCommands(item, OperationType.Put);
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
if (_script.HasLoadCounterBehaviors && _script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, function);
}
}
}
else
{
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, GetAttachmentsFor(item), GetCounterOperationsFor(item));
}
break;
case EtlItemType.CounterGroup:
if (_script.HasTransformation)
{
if (_script.HasLoadCounterBehaviors == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var function) == false)
{
break;
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, function);
}
else
{
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument);
}
break;
}
}
else
{
Debug.Assert(item.Type == EtlItemType.Document);
if (ShouldFilterOutDeletion(item) == false)
{
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
}
}
_commands.AddRange(_currentRun.GetCommands());
}