private IEnumerable <Document> GetDocumentFromCollectionThatReference(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, string collection, Reference referencedDocument, long lastIndexedEtag)
{
foreach (var key in _indexStorage.GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction))
{
using (GetLower(out Slice loweredKey))
{
// when there is conflict, we need to apply same behavior as if the document would not exist
var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);
if (doc == null)
{
continue;
}
if (doc.Etag > lastIndexedEtag)
{
doc.Dispose();
continue;
}
if (doc.Etag > referencedDocument.Etag)
{
//IF the map worker already mapped this "doc" version it must be with this version of "referencedDocument" and if the map worker didn't mapped the "doc" so it will process it later
doc.Dispose();
continue;
}
yield return(doc);
}
unsafe ByteStringContext <ByteStringMemoryCache> .InternalScope GetLower(out Slice loweredKey)
{
return(DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out loweredKey));
}
}
}
public void DeleteRevisionsFor(DocumentsOperationContext context, string id)
{
using (DocumentIdWorker.GetSliceFromId(context, id, out Slice lowerId))
using (GetKeyPrefix(context, lowerId, out Slice prefixSlice))
{
var collectionName = GetCollectionFor(context, prefixSlice);
if (collectionName == null)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Tried to delete all revisions for '{id}' but no revisions found.");
}
return;
}
var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);
var newEtag = _documentsStorage.GenerateNextEtag();
var changeVector = _documentsStorage.GetNewChangeVector(context, newEtag);
context.LastDatabaseChangeVector = changeVector;
var lastModifiedTicks = _database.Time.GetUtcNow().Ticks;
DeleteRevisions(context, table, prefixSlice, collectionName, long.MaxValue, null, changeVector, lastModifiedTicks);
DeleteCountOfRevisions(context, prefixSlice);
}
}
public IEnumerable <Slice> GetItemKeysFromCollectionThatReference(string collection, LazyStringValue referenceKey, RavenTransaction tx, string lastItemId = null)
{
var collectionTree = tx.InnerTransaction.ReadTree(_referenceCollectionPrefix + collection);
if (collectionTree == null)
{
yield break;
}
using (DocumentIdWorker.GetLower(tx.InnerTransaction.Allocator, referenceKey, out var k))
using (var it = collectionTree.MultiRead(k))
{
if (lastItemId == null)
{
if (it.Seek(Slices.BeforeAllKeys) == false)
{
yield break;
}
}
else
{
using (Slice.From(tx.InnerTransaction.Allocator, lastItemId, out var idSlice))
{
if (it.Seek(idSlice) == false)
{
yield break;
}
}
}
do
{
yield return(it.CurrentKey);
} while (it.MoveNext());
}
}
public bool Update(UpdateStep step)
{
// Update collections
using (step.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var readTable = step.ReadTx.OpenTable(CollectionsSchema, CollectionsSlice);
if (readTable != null)
{
var writeTable = step.WriteTx.OpenTable(CollectionsSchema, CollectionsSlice);
foreach (var read in readTable.SeekByPrimaryKey(Slices.BeforeAllKeys, 0))
{
using (TableValueReaderUtil.CloneTableValueReader(context, read))
{
var collection = TableValueToString(context, (int)CollectionsTable.Name, ref read.Reader);
using (DocumentIdWorker.GetStringPreserveCase(context, collection, out Slice collectionSlice))
using (writeTable.Allocate(out TableValueBuilder write))
{
write.Add(collectionSlice);
var pk = read.Reader.Read((int)CollectionsTable.Name, out int size);
using (Slice.External(context.Allocator, pk, size, out var pkSlice))
{
writeTable.DeleteByKey(pkSlice);
}
writeTable.Insert(write);
}
}
}
}
}
// Update tombstones's collection value
using (step.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
foreach (var collection in step.DocumentsStorage.GetTombstoneCollections(step.ReadTx))
{
string tableName;
if (collection == AttachmentsTombstones ||
collection == RevisionsTombstones)
{
tableName = collection;
}
else
{
var collectionName = new CollectionName(collection);
tableName = collectionName.GetTableName(CollectionTableType.Tombstones);
}
var readTable = step.ReadTx.OpenTable(TombstonesSchema, tableName);
if (readTable == null)
{
continue;
}
var writeTable = step.WriteTx.OpenTable(TombstonesSchema, tableName);
// We seek by an index instead the PK because
// we weed to ensure that we aren't accessing an IsGlobal key
foreach (var read in readTable.SeekForwardFrom(TombstonesSchema.FixedSizeIndexes[CollectionEtagsSlice], 0, 0))
{
// We copy the memory of the read so AssertNoReferenceToOldData won't throw.
// This is done instead of moving AssertNoReferenceToOldData to assert later
// after we allocate the new write memory.
using (TableValueReaderUtil.CloneTableValueReader(context, read))
{
var type = *(Tombstone.TombstoneType *)read.Reader.Read((int)TombstoneTable.Type, out _);
var oldCollection = TableValueToString(context, (int)TombstoneTable.Collection, ref read.Reader);
using (DocumentIdWorker.GetStringPreserveCase(context, oldCollection, out Slice collectionSlice))
using (writeTable.Allocate(out TableValueBuilder write))
{
write.Add(read.Reader.Read((int)TombstoneTable.LowerId, out int size), size);
write.Add(read.Reader.Read((int)TombstoneTable.Etag, out size), size);
write.Add(read.Reader.Read((int)TombstoneTable.DeletedEtag, out size), size);
write.Add(read.Reader.Read((int)TombstoneTable.TransactionMarker, out size), size);
write.Add(read.Reader.Read((int)TombstoneTable.Type, out size), size);
if (type == Tombstone.TombstoneType.Attachment)
{
write.Add(read.Reader.Read((int)TombstoneTable.Collection, out size), size);
}
else
{
write.Add(collectionSlice);
}
write.Add(read.Reader.Read((int)TombstoneTable.Flags, out size), size);
write.Add(read.Reader.Read((int)TombstoneTable.ChangeVector, out size), size);
write.Add(read.Reader.Read((int)TombstoneTable.LastModified, out size), size);
writeTable.Set(write);
}
}
}
}
}
// Update conflicts' collection value
using (step.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var readTable = step.ReadTx.OpenTable(ConflictsSchema, ConflictsSlice);
if (readTable != null)
{
var writeTable = step.WriteTx.OpenTable(ConflictsSchema, ConflictsSlice);
foreach (var read in readTable.SeekByPrimaryKey(Slices.BeforeAllKeys, 0))
{
using (TableValueReaderUtil.CloneTableValueReader(context, read))
{
var oldCollection = TableValueToString(context, (int)ConflictsTable.Collection, ref read.Reader);
using (DocumentIdWorker.GetStringPreserveCase(context, oldCollection, out Slice collectionSlice))
using (writeTable.Allocate(out TableValueBuilder write))
{
write.Add(read.Reader.Read((int)ConflictsTable.LowerId, out int size), size);
write.Add(read.Reader.Read((int)ConflictsTable.RecordSeparator, out size), size);
write.Add(read.Reader.Read((int)ConflictsTable.ChangeVector, out size), size);
write.Add(read.Reader.Read((int)ConflictsTable.Id, out size), size);
write.Add(read.Reader.Read((int)ConflictsTable.Data, out size), size);
write.Add(read.Reader.Read((int)ConflictsTable.Etag, out size), size);
write.Add(collectionSlice);
write.Add(read.Reader.Read((int)ConflictsTable.LastModified, out size), size);
write.Add(read.Reader.Read((int)ConflictsTable.Flags, out size), size);
writeTable.Set(write);
}
}
}
}
}
return(true);
}
private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
{
var moreWorkFound = false;
Dictionary <string, long> lastIndexedEtagsByCollection = null;
foreach (var collection in _index.Collections)
{
if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
if (lastIndexedEtagsByCollection == null)
{
lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
}
if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
{
lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
{
continue;
}
foreach (var referencedCollection in referencedCollections)
{
using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}.");
}
long lastReferenceEtag;
switch (actionType)
{
case ActionType.Document:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection);
break;
case ActionType.Tombstone:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection);
break;
default:
throw new NotSupportedException();
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}.");
}
var lastEtag = lastReferenceEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
IEnumerable <Reference> references;
switch (actionType)
{
case ActionType.Document:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(document =>
{
_reference.Key = document.Id;
_reference.Etag = document.Etag;
return(_reference);
});
break;
case ActionType.Tombstone:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(tombstone =>
{
_reference.Key = tombstone.LowerId;
_reference.Etag = tombstone.Etag;
return(_reference);
});
break;
default:
throw new NotSupportedException();
}
foreach (var referencedDocument in references)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}.");
}
lastEtag = referencedDocument.Etag;
count++;
batchCount++;
var documents = new List <Document>();
foreach (var key in _indexStorage
.GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction))
{
using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
{
// when there is conflict, we need to apply same behavior as if the document would not exist
var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);
if (doc != null && doc.Etag <= lastIndexedEtag)
{
documents.Add(doc);
}
}
}
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (docsEnumerator.MoveNext(out IEnumerable mapResults))
{
token.ThrowIfCancellationRequested();
var current = docsEnumerator.Current;
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}.");
}
try
{
_index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats);
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
}
}
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
switch (actionType)
{
case ActionType.Document:
_indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
case ActionType.Tombstone:
_indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
default:
throw new NotSupportedException();
}
moreWorkFound = true;
}
}
}
return(moreWorkFound);
}
public override void HandleDelete(Tombstone tombstone, string collection, Lazy <IndexWriteOperation> writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
using (DocumentIdWorker.GetSliceFromId(indexContext, tombstone.LowerId, out Slice documentIdPrefixWithTsKeySeparator, SpecialChars.RecordSeparator))
_referencesStorage.RemoveReferencesByPrefix(documentIdPrefixWithTsKeySeparator, collection, null, indexContext.Transaction);
}
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document,
DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks,
RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null)
{
Debug.Assert(changeVector != null, "Change vector must be set");
Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set");
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
if (collectionName == null)
{
collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document);
}
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler;
var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication;
var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);
// We want the revision's attachments to have a lower etag than the revision itself
if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments &&
fromSmuggler == false)
{
using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice))
{
if (table.VerifyKeyExists(changeVectorSlice) == false)
{
_documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice);
}
}
}
if (fromReplication)
{
void PutFromRevisionIfChangeVectorIsGreater()
{
bool hasDoc;
TableValueReader tvr;
try
{
hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr);
}
catch (DocumentConflictException)
{
// Do not modify the document.
return;
}
if (hasDoc == false)
{
PutFromRevision();
return;
}
var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr);
if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update)
{
PutFromRevision();
}
void PutFromRevision()
{
_documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector,
flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision);
}
}
PutFromRevisionIfChangeVectorIsGreater();
}
flags |= DocumentFlags.Revision;
var newEtag = _database.DocumentsStorage.GenerateNextEtag();
var newEtagSwapBytes = Bits.SwapBytes(newEtag);
using (table.Allocate(out TableValueBuilder tvb))
using (Slice.From(context.Allocator, changeVector, out var cv))
{
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(newEtagSwapBytes);
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add((int)flags);
tvb.Add(NotDeletedRevisionMarker);
tvb.Add(lastModifiedTicks);
tvb.Add(context.GetTransactionMarker());
if (flags.Contain(DocumentFlags.Resolved))
{
tvb.Add((int)DocumentFlags.Resolved);
}
else
{
tvb.Add(0);
}
tvb.Add(Bits.SwapBytes(lastModifiedTicks));
var isNew = table.Set(tvb);
if (isNew == false)
{
// It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision.
return;
}
}
if (configuration == null)
{
configuration = GetRevisionsConfiguration(collectionName.Name);
}
DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks);
}
}
public unsafe void HandleConflictForDocument(
DocumentsOperationContext documentsContext,
string id,
string collection,
long lastModifiedTicks,
BlittableJsonReaderObject doc,
string changeVector,
DocumentFlags flags)
{
if (id.StartsWith(HiLoHandler.RavenHiloIdPrefix, StringComparison.OrdinalIgnoreCase))
{
HandleHiloConflict(documentsContext, id, doc, changeVector);
return;
}
if (TryResolveIdenticalDocument(
documentsContext,
id,
doc,
lastModifiedTicks,
changeVector))
{
return;
}
var lazyId = documentsContext.GetLazyString(id);
using (DocumentIdWorker.GetLower(documentsContext.Allocator, lazyId, out var loweredKey))
{
var conflictedDoc = new DocumentConflict
{
Doc = doc,
Collection = documentsContext.GetLazyStringForFieldWithCaching(
collection ??
CollectionName.GetCollectionName(doc)
),
LastModified = new DateTime(lastModifiedTicks),
LowerId = documentsContext.AllocateStringValue(null, loweredKey.Content.Ptr, loweredKey.Content.Length),
Id = lazyId,
ChangeVector = changeVector,
Flags = flags
};
if (TryResolveConflictByScript(
documentsContext,
conflictedDoc))
{
return;
}
if (_conflictResolver.ConflictSolver?.ResolveToLatest ?? true)
{
var conflicts = new List <DocumentConflict>
{
conflictedDoc.Clone()
};
conflicts.AddRange(_database.DocumentsStorage.ConflictsStorage.GetConflictsFor(documentsContext, id));
var localDocumentTuple = _database.DocumentsStorage.GetDocumentOrTombstone(documentsContext, id, false);
var local = DocumentConflict.From(documentsContext, localDocumentTuple.Document) ?? DocumentConflict.From(localDocumentTuple.Tombstone);
if (local != null)
{
conflicts.Add(local);
}
var resolved = _conflictResolver.ResolveToLatest(conflicts);
_conflictResolver.PutResolvedDocument(documentsContext, resolved, resolvedToLatest: true, conflictedDoc);
return;
}
_database.DocumentsStorage.ConflictsStorage.AddConflict(documentsContext, id, lastModifiedTicks, doc, changeVector, collection, flags);
}
}
public override int Execute(DocumentsOperationContext context)
{
if (Database.ServerStore.Configuration.Core.FeaturesAvailability == FeaturesAvailability.Stable)
{
FeaturesAvailabilityException.Throw("Cluster Transactions");
}
var global = context.LastDatabaseChangeVector ??
(context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context));
var dbGrpId = Database.DatabaseGroupId;
var current = ChangeVectorUtils.GetEtagById(global, dbGrpId);
foreach (var command in _batch)
{
Replies.Add(command.Index, new DynamicJsonArray());
Reply = Replies[command.Index];
var commands = command.Commands;
var count = command.PreviousCount;
var options = Options[command.Index] = command.Options;
if (options.WaitForIndexesTimeout != null)
{
ModifiedCollections = new HashSet <string>();
}
if (commands != null)
{
foreach (BlittableJsonReaderObject blittableCommand in commands)
{
count++;
var changeVector = $"RAFT:{count}-{dbGrpId}";
var cmd = JsonDeserializationServer.ClusterTransactionDataCommand(blittableCommand);
switch (cmd.Type)
{
case CommandType.PUT:
if (current < count)
{
// delete the document to avoid exception if we put new document in a different collection.
// TODO: document this behavior
using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId))
{
Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, expectedChangeVector: null,
nonPersistentFlags: NonPersistentDocumentFlags.SkipRevisionCreation);
}
var putResult = Database.DocumentsStorage.Put(context, cmd.Id, null, cmd.Document.Clone(context), changeVector: changeVector,
flags: DocumentFlags.FromClusterTransaction);
context.DocumentDatabase.HugeDocuments.AddIfDocIsHuge(cmd.Id, cmd.Document.Size);
AddPutResult(putResult);
}
else
{
try
{
var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id);
if (item.Missing)
{
AddPutResult(new DocumentsStorage.PutOperationResults
{
ChangeVector = changeVector,
Id = cmd.Id,
LastModified = DateTime.UtcNow,
Collection = Database.DocumentsStorage.ExtractCollectionName(context, cmd.Document)
});
continue;
}
var collection = GetCollection(context, item);
AddPutResult(new DocumentsStorage.PutOperationResults
{
ChangeVector = changeVector,
Id = cmd.Id,
Flags = item.Document?.Flags ?? item.Tombstone.Flags,
LastModified = item.Document?.LastModified ?? item.Tombstone.LastModified,
Collection = collection
});
}
catch (DocumentConflictException)
{
AddPutResult(new DocumentsStorage.PutOperationResults
{
ChangeVector = changeVector,
Id = cmd.Id,
Collection = GetFirstConflictCollection(context, cmd)
});
}
}
break;
case CommandType.DELETE:
if (current < count)
{
using (DocumentIdWorker.GetSliceFromId(context, cmd.Id, out Slice lowerId))
{
var deleteResult = Database.DocumentsStorage.Delete(context, lowerId, cmd.Id, null, changeVector: changeVector,
documentFlags: DocumentFlags.FromClusterTransaction);
AddDeleteResult(deleteResult, cmd.Id);
}
}
else
{
try
{
var item = Database.DocumentsStorage.GetDocumentOrTombstone(context, cmd.Id);
if (item.Missing)
{
AddDeleteResult(new DocumentsStorage.DeleteOperationResult
{
ChangeVector = changeVector,
Collection = null
}, cmd.Id);
continue;
}
var collection = GetCollection(context, item);
AddDeleteResult(new DocumentsStorage.DeleteOperationResult
{
ChangeVector = changeVector,
Collection = collection
}, cmd.Id);
}
catch (DocumentConflictException)
{
AddDeleteResult(new DocumentsStorage.DeleteOperationResult
{
ChangeVector = changeVector,
Collection = GetFirstConflictCollection(context, cmd)
}, cmd.Id);
}
}
break;
default:
throw new NotSupportedException($"{cmd.Type} is not supported in {nameof(ClusterTransactionMergedCommand)}.");
}
}
}
if (context.LastDatabaseChangeVector == null)
{
context.LastDatabaseChangeVector = global;
}
var result = ChangeVectorUtils.TryUpdateChangeVector("RAFT", dbGrpId, count, context.LastDatabaseChangeVector);
if (result.IsValid)
{
context.LastDatabaseChangeVector = result.ChangeVector;
}
}
return(Reply.Count);
}
private CollectionName PutCounters(UpdateStep step, DocumentsOperationContext context, HashSet <string> dbIds, Dictionary <string, List <CounterDetail> > allCountersBatch, string docId)
{
string collection = null;
using (DocumentIdWorker.GetSliceFromId(context, docId, out Slice lowerId))
{
var docsTable = new Table(DocsSchema, step.ReadTx);
if (docsTable.ReadByKey(lowerId, out var tvr))
{
using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context))
{
collection = CollectionName.GetCollectionName(doc);
}
}
else
{
// document does not exist
return(null);
}
}
var collectionName = new CollectionName(collection);
using (DocumentIdWorker.GetSliceFromId(context, docId, out Slice documentKeyPrefix, separator: SpecialChars.RecordSeparator))
{
var maxNumberOfCountersPerGroup = Math.Max(32, 2048 / (dbIds.Count * 32 + 1)); // rough estimate
var orderedKeys = allCountersBatch.OrderBy(x => x.Key).ToList();
var listOfDbIds = dbIds.ToList();
for (int i = 0; i < orderedKeys.Count / maxNumberOfCountersPerGroup + (orderedKeys.Count % maxNumberOfCountersPerGroup == 0 ? 0 : 1); i++)
{
var currentBatch = allCountersBatch.Skip(maxNumberOfCountersPerGroup * i).Take(maxNumberOfCountersPerGroup);
using (var data = WriteNewCountersDocument(context, listOfDbIds, currentBatch))
{
var etag = step.DocumentsStorage.GenerateNextEtag();
var changeVector = ChangeVectorUtils.NewChangeVector(
step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, etag, _dbId);
var table = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName);
data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject values);
BlittableJsonReaderObject.PropertyDetails prop = default;
values.GetPropertyByIndex(0, ref prop);
using (table.Allocate(out TableValueBuilder tvb))
{
using (Slice.From(context.Allocator, changeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice))
using (context.Allocator.Allocate(documentKeyPrefix.Size + prop.Name.Size, out var counterKeyBuffer))
using (Slice.From(context.Allocator, prop.Name, out var nameSlice))
using (CreateCounterKeySlice(context, counterKeyBuffer, documentKeyPrefix, nameSlice, out var counterKeySlice))
{
if (i == 0)
{
tvb.Add(documentKeyPrefix);
}
else
{
tvb.Add(counterKeySlice);
}
tvb.Add(Bits.SwapBytes(etag));
tvb.Add(cv);
tvb.Add(data.BasePointer, data.Size);
tvb.Add(collectionSlice);
tvb.Add(context.GetTransactionMarker());
table.Set(tvb);
}
}
}
}
}
return(collectionName);
}
private void DeleteCounter(UpdateStep step, LazyStringValue tombstoneKey, DocumentsOperationContext context)
{
var(docId, counterName) = ExtractDocIdAndNameFromCounterTombstone(context, tombstoneKey);
using (docId)
using (counterName)
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, docId, out Slice lowerId, out _))
{
string collection = null;
var docsTable = new Table(DocsSchema, step.ReadTx);
if (docsTable.ReadByKey(lowerId, out var tvr))
{
using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context))
{
collection = CollectionName.GetCollectionName(doc);
}
}
var collectionName = new CollectionName(collection);
var table = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName);
if (table.ReadByKey(lowerId, out var existing) == false)
{
return;
}
// (int)CountersTable.Data = 3
var data = new BlittableJsonReaderObject(existing.Read(3, out int oldSize), oldSize, context);
if (data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject counters) == false ||
counters.TryGetMember(counterName, out object counterToDelete) == false ||
counterToDelete is LazyStringValue) // already deleted
{
return;
}
var deleteCv = step.DocumentsStorage.CountersStorage.GenerateDeleteChangeVectorFromRawBlob(data, counterToDelete as BlittableJsonReaderObject.RawBlob);
counters.Modifications = new DynamicJsonValue(counters)
{
[counterName] = deleteCv
};
using (var old = data)
{
data = context.ReadObject(data, null, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
var newEtag = step.DocumentsStorage.GenerateNextEtag();
var newChangeVector = ChangeVectorUtils.NewChangeVector(step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, newEtag, _dbId);
using (data)
using (Slice.From(context.Allocator, newChangeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(cv);
tvb.Add(data.BasePointer, data.Size);
tvb.Add(collectionSlice);
tvb.Add(context.GetTransactionMarker());
table.Set(tvb);
}
}
}
public unsafe dynamic LoadDocument(LazyStringValue keyLazy, string keyString, string collectionName)
{
using (_loadDocumentStats?.Start() ?? (_loadDocumentStats = _stats?.For(IndexingOperation.LoadDocument)))
{
if (keyLazy == null && keyString == null)
{
return(DynamicNullObject.Null);
}
var source = Source;
if (source == null)
{
throw new ArgumentException("Cannot execute LoadDocument. Source is not set.");
}
var id = source.GetId() as LazyStringValue;
if (id == null)
{
throw new ArgumentException("Cannot execute LoadDocument. Source does not have a key.");
}
if (keyLazy != null && id.Equals(keyLazy))
{
return(source);
}
if (keyString != null && id.Equals(keyString))
{
return(source);
}
// we intentionally don't dispose of the scope here, this is being tracked by the references
// and will be disposed there.
// making sure that we normalize the case of the key so we'll be able to find
// it in case insensitive manner
// In addition, special characters need to be escaped
Slice keySlice;
if (keyLazy != null)
{
if (keyLazy.Length == 0)
{
return(DynamicNullObject.Null);
}
DocumentIdWorker.GetSliceFromId(_documentsContext, keyLazy, out keySlice);
}
else
{
if (keyString.Length == 0)
{
return(DynamicNullObject.Null);
}
DocumentIdWorker.GetSliceFromId(_documentsContext, keyString, out keySlice);
}
var references = GetReferencesForDocument(id);
references.Add(keySlice);
// when there is conflict, we need to apply same behavior as if the document would not exist
var document = _documentsStorage.Get(_documentsContext, keySlice, throwOnConflict: false);
if (document == null)
{
return(DynamicNullObject.Null);
}
// we can't share one DynamicBlittableJson instance among all documents because we can have multiple LoadDocuments in a single scope
return(new DynamicBlittableJson(document));
}
}
