public void UpdateHistoryLog(TransactionOperationContext context, long index, long term, BlittableJsonReaderObject cmd, object result, Exception exception)
{
var guid = GetGuidFromCommand(cmd);
if (guid == null) // shouldn't happened in new cluster version!
{
return;
}
if (guid == RaftIdGenerator.DontCareId)
{
return;
}
var type = GetTypeFromCommand(cmd);
UpdateInternal(context, guid, type, index, term, HistoryStatus.Committed, result, exception);
}
public override BlittableJsonReaderObject GetUpdatedValue(JsonOperationContext context, BlittableJsonReaderObject previousValue, long index)
{
if (Value == null)
{
throw new RachisApplyException($"{nameof(Value)} cannot be null");
}
if (Value.DetailsPerNode == null || Value.NodeTag == null || Value.AllNodes == null)
{
throw new RachisApplyException($"{nameof(Value.DetailsPerNode)}, {nameof(Value.NodeTag)}, {nameof(Value.AllNodes)} cannot be null");
}
if (Value.DetailsPerNode.MaxUtilizedCores != null && Value.DetailsPerNode.MaxUtilizedCores <= 0)
{
throw new RachisApplyException($"{nameof(DetailsPerNode.MaxUtilizedCores)} must be greater than 0");
}
var licenseLimits = previousValue == null ? new LicenseLimits() : JsonDeserializationServer.LicenseLimits(previousValue);
RemoveNotInClusterNodes(licenseLimits);
UpdateNodeDetails(licenseLimits);
ReBalanceCores(licenseLimits);
return(context.ReadObject(licenseLimits.ToJson(), "update-license-limits"));
}
public static LazyStringValue GetLazyCollectionNameFrom(JsonOperationContext context, BlittableJsonReaderObject document)
{
if (document.TryGet(MetadataKeySegment, out BlittableJsonReaderObject metadata) == false ||
metadata.TryGet(MetadataCollectionSegment, out LazyStringValue collectionName) == false)
{
return(context.GetLazyStringForFieldWithCaching(EmptyCollectionSegment));
}
return(collectionName);
}
public override void VisitBetween(QueryExpression fieldName, QueryExpression firstValue, QueryExpression secondValue, BlittableJsonReaderObject parameters)
{
if (fieldName is MethodExpression me && string.Equals("id", me.Name, StringComparison.OrdinalIgnoreCase) && firstValue is ValueExpression fv && secondValue is ValueExpression sv)
{
throw new InvalidQueryException("Collection query does not support filtering by id() using Between operator. Supported operators are: =, IN",
QueryText, parameters);
}
}
public abstract Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, QueryOperationContext queryContext, Action <IOperationProgress> onProgress, OperationCancelToken token);
private bool ShouldRecreateAttachments(DocumentsOperationContext context, Slice lowerId, BlittableJsonReaderObject oldDoc,
BlittableJsonReaderObject document, ref DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags)
{
if ((nonPersistentFlags & NonPersistentDocumentFlags.ResolveAttachmentsConflict) == NonPersistentDocumentFlags.ResolveAttachmentsConflict)
{
document.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata);
RecreateAttachments(context, lowerId, document, metadata, ref flags);
return(true);
}
if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments &&
(nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if (oldDoc != null &&
oldDoc.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject oldMetadata) &&
oldMetadata.TryGet(Constants.Documents.Metadata.Attachments, out BlittableJsonReaderArray oldAttachments))
{
// Make sure the user did not changed the value of @attachments in the @metadata
// In most cases it won't be changed so we can use this value
// instead of recreating the document's blitable from scratch
if (document.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) == false ||
metadata.TryGet(Constants.Documents.Metadata.Attachments, out BlittableJsonReaderArray attachments) == false ||
attachments.Equals(oldAttachments) == false)
{
RecreateAttachments(context, lowerId, document, metadata, ref flags);
return(true);
}
}
}
return(false);
}
public override void VisitBooleanMethod(QueryExpression leftSide, QueryExpression rightSide, OperatorType operatorType, BlittableJsonReaderObject parameters)
{
VisitFieldToken(leftSide, rightSide, parameters, operatorType);
}
private void HandleAttachmentDelete(BlittableJsonReaderObject batchResult)
{
HandleAttachmentDeleteInternal(batchResult, CommandType.AttachmentDELETE, Constants.Documents.Metadata.Id, nameof(DeleteAttachmentCommandData.Name), nameof(Constants.Fields.CommandData.DocumentChangeVector));
}
private void HandleAttachmentPut(BlittableJsonReaderObject batchResult)
{
HandleAttachmentPutInternal(batchResult, CommandType.AttachmentPUT, nameof(PutAttachmentCommandData.Id), nameof(PutAttachmentCommandData.Name), nameof(Constants.Fields.CommandData.DocumentChangeVector));
}
private void HandleCompareExchangeDelete(BlittableJsonReaderObject batchResult)
{
HandleCompareExchangeInternal(CommandType.CompareExchangeDELETE, batchResult);
}
private void HandleAttachmentMove(BlittableJsonReaderObject batchResult)
{
HandleAttachmentDeleteInternal(batchResult, CommandType.AttachmentMOVE, nameof(MoveAttachmentCommandData.Id), nameof(MoveAttachmentCommandData.Name), nameof(Constants.Fields.CommandData.DocumentChangeVector));
HandleAttachmentPutInternal(batchResult, CommandType.AttachmentMOVE, nameof(MoveAttachmentCommandData.DestinationId), nameof(MoveAttachmentCommandData.DestinationName), nameof(Constants.Fields.CommandData.DestinationDocumentChangeVector));
}
private void HandleCompareExchangePut(BlittableJsonReaderObject batchResult)
{
HandleCompareExchangeInternal(CommandType.CompareExchangePUT, batchResult);
}
public int SizeAfterAdding(BlittableJsonReaderObject item)
{
return(_dataSize + item.Size + sizeof(ResultHeader));
}
public unsafe bool HasHistoryLog(TransactionOperationContext context, BlittableJsonReaderObject cmd, out long index, out object result, out Exception exception)
{
result = null;
exception = null;
index = 0;
var guid = GetGuidFromCommand(cmd);
if (guid == null) // shouldn't happened in new cluster version!
{
return(false);
}
if (guid == RaftIdGenerator.DontCareId)
{
return(false);
}
var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice);
using (Slice.From(context.Allocator, guid, out var guidSlice))
{
if (table.ReadByKey(guidSlice, out var reader) == false)
{
return(false);
}
index = Bits.SwapBytes(*(long *)reader.Read((int)LogHistoryColumn.Index, out _));
var resultPtr = reader.Read((int)LogHistoryColumn.Result, out int size);
if (size > 0)
{
var blittableResultHolder = new BlittableJsonReaderObject(resultPtr, size, context);
if (blittableResultHolder.TryGet("Result", out result) == false) // this blittable will be clone to the outer context
{
throw new InvalidOperationException("Invalid result format!"); // shouldn't happened!
}
}
var exceptionTypePtr = reader.Read((int)LogHistoryColumn.ExceptionType, out size);
if (size > 0)
{
var exceptionMsgPtr = reader.Read((int)LogHistoryColumn.ExceptionType, out size);
var exceptionMsg = Encodings.Utf8.GetString(exceptionMsgPtr, size);
try
{
var exceptionType = Type.GetType(Encodings.Utf8.GetString(exceptionTypePtr, size));
exception = (Exception)Activator.CreateInstance(exceptionType, exceptionMsg);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"failed to generate the exception dynamically for guid {guid}", e);
}
exception = new Exception($"failed to generate the exception dynamically, but the original exception message is:" +
$"{Environment.NewLine}{exceptionMsg}");
}
}
return(true);
}
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = _documentsStorage.GetOrCreateLastModifiedTicks(lastModifiedTicks);
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care,
// and empty means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
if (string.IsNullOrEmpty(result.ChangeVector))
{
ChangeVectorUtils.ThrowConflictingEtag(id, changeVector, newEtag, _documentsStorage.Environment.Base64Id, _documentDatabase.ServerStore.NodeTag);
}
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}
private void HandleDelete(BlittableJsonReaderObject batchResult)
{
HandleDeleteInternal(batchResult, CommandType.DELETE);
}
private void RecreateAttachments(DocumentsOperationContext context, Slice lowerId, BlittableJsonReaderObject document,
BlittableJsonReaderObject metadata, ref DocumentFlags flags)
{
var actualAttachments = _documentsStorage.AttachmentsStorage.GetAttachmentsMetadataForDocument(context, lowerId);
if (actualAttachments.Count == 0)
{
if (metadata != null)
{
metadata.Modifications = new DynamicJsonValue(metadata);
metadata.Modifications.Remove(Constants.Documents.Metadata.Attachments);
document.Modifications = new DynamicJsonValue(document)
{
[Constants.Documents.Metadata.Key] = metadata
};
}
flags &= ~DocumentFlags.HasAttachments;
return;
}
flags |= DocumentFlags.HasAttachments;
if (metadata == null)
{
document.Modifications = new DynamicJsonValue(document)
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Attachments] = actualAttachments
}
};
}
else
{
metadata.Modifications = new DynamicJsonValue(metadata)
{
[Constants.Documents.Metadata.Attachments] = actualAttachments
};
document.Modifications = new DynamicJsonValue(document)
{
[Constants.Documents.Metadata.Key] = metadata
};
}
}
public Task <IOperationResult> ExecutePatch(string collectionName, long start, long take, CollectionOperationOptions options, PatchRequest patch,
BlittableJsonReaderObject patchArgs, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(ExecuteOperation(collectionName, start, take, options, Context, onProgress,
key => new PatchDocumentCommand(Context, key, null, false, (patch, patchArgs), (null, null), Database, false, false, false, false), token));
}
public async Task Patch()
{
var id = GetQueryStringValueAndAssertIfSingleAndNotEmpty("id");
var isTest = GetBoolValueQueryString("test", required: false) ?? false;
var debugMode = GetBoolValueQueryString("debug", required: false) ?? isTest;
var skipPatchIfChangeVectorMismatch = GetBoolValueQueryString("skipPatchIfChangeVectorMismatch", required: false) ?? false;
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var request = context.Read(RequestBodyStream(), "ScriptedPatchRequest");
if (request.TryGet("Patch", out BlittableJsonReaderObject patchCmd) == false || patchCmd == null)
{
throw new ArgumentException("The 'Patch' field in the body request is mandatory");
}
var patch = PatchRequest.Parse(patchCmd, out var patchArgs);
PatchRequest patchIfMissing = null;
BlittableJsonReaderObject patchIfMissingArgs = null;
if (request.TryGet("PatchIfMissing", out BlittableJsonReaderObject patchIfMissingCmd) && patchIfMissingCmd != null)
{
patchIfMissing = PatchRequest.Parse(patchIfMissingCmd, out patchIfMissingArgs);
}
var changeVector = context.GetLazyString(GetStringFromHeaders("If-Match"));
using (var command = new PatchDocumentCommand(context,
id,
changeVector,
skipPatchIfChangeVectorMismatch,
(patch, patchArgs),
(patchIfMissing, patchIfMissingArgs),
Database,
isTest,
debugMode,
true
))
{
if (isTest == false)
{
await Database.TxMerger.Enqueue(command);
}
else
{
// PutDocument requires the write access to the docs storage
// testing patching is rare enough not to optimize it
using (context.OpenWriteTransaction())
{
command.Execute(context);
}
}
switch (command.PatchResult.Status)
{
case PatchStatus.DocumentDoesNotExist:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
return;
case PatchStatus.Created:
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
break;
case PatchStatus.Skipped:
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return;
case PatchStatus.Patched:
case PatchStatus.NotModified:
HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
break;
default:
throw new ArgumentOutOfRangeException();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName(nameof(command.PatchResult.Status));
writer.WriteString(command.PatchResult.Status.ToString());
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.ModifiedDocument));
writer.WriteObject(command.PatchResult.ModifiedDocument);
if (debugMode)
{
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.OriginalDocument));
if (isTest)
{
writer.WriteObject(command.PatchResult.OriginalDocument);
}
else
{
writer.WriteNull();
}
writer.WriteComma();
writer.WritePropertyName(nameof(command.PatchResult.Debug));
context.Write(writer, new DynamicJsonValue
{
["Info"] = new DynamicJsonArray(command.DebugOutput),
["Actions"] = command.DebugActions?.GetDebugActions()
});
}
writer.WriteEndObject();
}
}
}
}
protected Field GetOrCreateField(string name, string value, LazyStringValue lazyValue, BlittableJsonReaderObject blittableValue, Field.Store store, Field.Index index, Field.TermVector termVector)
{
int cacheKey = FieldCacheKey.GetHashCode(name, index, store, termVector, _multipleItemsSameFieldCount);
Field field;
if (_fieldsCache.TryGetValue(cacheKey, out CachedFieldItem <Field> cached) == false ||
!cached.Key.IsSame(name, index, store, termVector, _multipleItemsSameFieldCount))
{
LazyStringReader stringReader = null;
BlittableObjectReader blittableReader = null;
if ((lazyValue != null || blittableValue != null) && store.IsStored() == false && index.IsIndexed() && index.IsAnalyzed())
{
TextReader reader;
if (lazyValue != null)
{
stringReader = new LazyStringReader();
reader = stringReader.GetTextReaderFor(lazyValue);
}
else
{
blittableReader = new BlittableObjectReader();
reader = blittableReader.GetTextReaderFor(blittableValue);
}
field = new Field(name, reader, termVector);
}
else
{
if (value == null && lazyValue == null)
{
blittableReader = new BlittableObjectReader();
}
field = new Field(name,
value ?? LazyStringReader.GetStringFor(lazyValue) ?? blittableReader.GetStringFor(blittableValue),
store, index, termVector);
}
field.Boost = 1;
field.OmitNorms = true;
_fieldsCache[cacheKey] = new CachedFieldItem <Field>
{
Key = new FieldCacheKey(name, index, store, termVector, _multipleItemsSameFieldCount.ToArray()),
Field = field,
LazyStringReader = stringReader,
BlittableObjectReader = blittableReader
};
}
else
{
field = cached.Field;
if (lazyValue != null && cached.LazyStringReader == null)
{
cached.LazyStringReader = new LazyStringReader();
}
if (blittableValue != null && cached.BlittableObjectReader == null)
{
cached.BlittableObjectReader = new BlittableObjectReader();
}
if ((lazyValue != null || blittableValue != null) && store.IsStored() == false && index.IsIndexed() && index.IsAnalyzed())
{
field.SetValue(cached.LazyStringReader?.GetTextReaderFor(lazyValue) ?? cached.BlittableObjectReader.GetTextReaderFor(blittableValue));
}
else
{
field.SetValue(value ?? LazyStringReader.GetStringFor(lazyValue) ?? cached.BlittableObjectReader.GetStringFor(blittableValue));
}
}
return(field);
}
public override void VisitFieldToken(QueryExpression fieldName, QueryExpression value, BlittableJsonReaderObject parameters, OperatorType?operatorType)
{
if (fieldName is MethodExpression me)
{
var methodType = QueryMethod.GetMethodType(me.Name);
switch (methodType)
{
case MethodType.Id:
if (value is ValueExpression ve)
{
var id = QueryBuilder.GetValue(_query, _metadata, parameters, ve);
Debug.Assert(id.Type == ValueTokenType.String || id.Type == ValueTokenType.Null);
AddId(id.Value?.ToString());
}
if (value is MethodExpression right)
{
var id = QueryBuilder.EvaluateMethod(_query, _metadata, _serverContext, _context, right, ref parameters);
if (id is ValueExpression v)
{
AddId(v.Token);
}
}
break;
}
}
}
public override Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, DocumentsOperationContext context, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
return(GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, context, onProgress, token));
}
public override void VisitIn(QueryExpression fieldName, List <QueryExpression> values, BlittableJsonReaderObject parameters)
{
if (Ids == null)
{
Ids = new HashSet <Slice>(SliceComparer.Instance); // this handles a case where IN is used with empty list
}
if (fieldName is MethodExpression me && string.Equals("id", me.Name, StringComparison.OrdinalIgnoreCase))
{
foreach (var item in values)
{
if (item is ValueExpression iv)
{
foreach (var id in QueryBuilder.GetValues(_query, _metadata, parameters, iv))
{
AddId(id.Value?.ToString());
}
}
}
}
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result)
{
result = null;
var shouldUpdateChangeVector = true;
var subscriptionName = SubscriptionName;
if (string.IsNullOrEmpty(subscriptionName))
{
subscriptionName = SubscriptionId.ToString();
}
//insert all docs to voron table. If exists, then batchId will be replaced
var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState);
var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, subscriptionName);
using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered))
using (Slice.From(context.Allocator, itemKey, out Slice valueName))
{
if (items.ReadByKey(valueNameLowered, out var tvr) == false)
{
throw new RachisApplyException($"Cannot find subscription {subscriptionName} @ {DatabaseName}");
}
var ptr = tvr.Read(2, out int size);
var existingValue = new BlittableJsonReaderObject(ptr, size, context);
if (existingValue == null)
{
throw new SubscriptionDoesNotExistException($"Subscription with name '{subscriptionName}' does not exist in database '{DatabaseName}'");
}
var subscriptionState = JsonDeserializationClient.SubscriptionState(existingValue);
var topology = record.Topology;
var lastResponsibleNode = AcknowledgeSubscriptionBatchCommand.GetLastResponsibleNode(HasHighlyAvailableTasks, topology, NodeTag);
var appropriateNode = topology.WhoseTaskIsIt(RachisState.Follower, subscriptionState, lastResponsibleNode);
if (appropriateNode == null && record.DeletionInProgress.ContainsKey(NodeTag))
{
throw new DatabaseDoesNotExistException($"Stopping subscription '{subscriptionName}' on node {NodeTag}, because database '{DatabaseName}' is being deleted.");
}
if (appropriateNode != NodeTag)
{
throw new SubscriptionDoesNotBelongToNodeException(
$"Cannot apply {nameof(AcknowledgeSubscriptionBatchCommand)} for subscription '{subscriptionName}' with id '{SubscriptionId}', on database '{DatabaseName}', on node '{NodeTag}'," +
$" because the subscription task belongs to '{appropriateNode ?? "N/A"}'.")
{
AppropriateNode = appropriateNode
};
}
if (CurrentChangeVector == nameof(Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange))
{
context.ReadObject(existingValue, subscriptionName);
shouldUpdateChangeVector = false;
}
if (subscriptionState.ChangeVectorForNextBatchStartingPoint != PreviouslyRecordedChangeVector)
{
throw new SubscriptionChangeVectorUpdateConcurrencyException($"Can't record subscription with name '{subscriptionName}' due to inconsistency in change vector progress. Probably there was an admin intervention that changed the change vector value. Stored value: {subscriptionState.ChangeVectorForNextBatchStartingPoint}, received value: {PreviouslyRecordedChangeVector}");
}
if (shouldUpdateChangeVector)
{
subscriptionState.ChangeVectorForNextBatchStartingPoint =
ChangeVectorUtils.MergeVectors(CurrentChangeVector, subscriptionState.ChangeVectorForNextBatchStartingPoint);
subscriptionState.NodeTag = NodeTag;
using (var obj = context.ReadObject(subscriptionState.ToJson(), "subscription"))
{
ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, obj);
}
}
}
foreach (var deletedId in Deleted)
{
using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndDocumentKey(context, DatabaseName, SubscriptionId, deletedId, out var key))
{
using var _ = Slice.External(context.Allocator, key, out var keySlice);
subscriptionStateTable.DeleteByKey(keySlice);
}
}
foreach (var documentRecord in Documents)
{
using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndDocumentKey(context, DatabaseName, SubscriptionId, documentRecord.DocumentId, out var key))
using (subscriptionStateTable.Allocate(out var tvb))
{
using var _ = Slice.External(context.Allocator, key, out var keySlice);
using var __ = Slice.From(context.Allocator, documentRecord.ChangeVector, out var changeVectorSlice);
tvb.Add(keySlice);
tvb.Add(changeVectorSlice);
tvb.Add(Bits.SwapBytes(index)); // batch id
subscriptionStateTable.Set(tvb);
}
}
foreach (var revisionRecord in Revisions)
{
using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndRevisionKey(context, DatabaseName, SubscriptionId, revisionRecord.Current, out var key))
using (subscriptionStateTable.Allocate(out var tvb))
{
using var _ = Slice.External(context.Allocator, key, out var keySlice);
using var __ = Slice.From(context.Allocator, revisionRecord.Previous ?? string.Empty, out var changeVectorSlice);
tvb.Add(keySlice);
tvb.Add(changeVectorSlice); //prev change vector
tvb.Add(Bits.SwapBytes(index)); // batch id
subscriptionStateTable.Set(tvb);
}
}
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, RawDatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue)
{
throw new NotImplementedException();
}
public DynamicBlittableJson(BlittableJsonReaderObject blittableJson)
{
BlittableJson = blittableJson;
}
public override async Task <IOperationResult> ExecutePatchQuery(IndexQueryServerSide query, QueryOperationOptions options, PatchRequest patch, BlittableJsonReaderObject patchArgs, QueryOperationContext queryContext, Action <IOperationProgress> onProgress, OperationCancelToken token)
{
Exception lastException = null;
for (var i = 0; i < NumberOfRetries; i++)
{
try
{
return(await GetRunner(query).ExecutePatchQuery(query, options, patch, patchArgs, queryContext, onProgress, token));
}
catch (ObjectDisposedException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
catch (OperationCanceledException e)
{
if (Database.DatabaseShutdown.IsCancellationRequested)
{
throw;
}
if (token.Token.IsCancellationRequested)
{
throw;
}
lastException = e;
await WaitForIndexBeingLikelyReplacedDuringQuery();
}
}
throw CreateRetriesFailedException(lastException);
}
public void Set(Document document)
{
_doc = document;
BlittableJson = document.Data;
}
public RawDatabaseRecord(BlittableJsonReaderObject record)
{
_record = record ?? throw new ArgumentNullException(nameof(record));
}
public void InsertHistoryLog(TransactionOperationContext context, long index, long term, BlittableJsonReaderObject cmd)
{
if (HasHistoryLog(context, cmd, out _, out _, out _))
{
return;
}
var guid = GetGuidFromCommand(cmd);
if (guid == null) // shouldn't happened in new cluster version!
{
return;
}
if (guid == RaftIdGenerator.DontCareId)
{
return;
}
var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice);
var type = GetTypeFromCommand(cmd);
using (Slice.From(context.Allocator, guid, out var guidSlice))
using (Slice.From(context.Allocator, type, out var typeSlice))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(guidSlice);
tvb.Add(Bits.SwapBytes(index));
tvb.Add(Bits.SwapBytes(GetUniqueTicks(context.Transaction.InnerTransaction)));
tvb.Add(term);
tvb.Add(0L);
tvb.Add(typeSlice);
tvb.Add((byte)HistoryStatus.Appended);
tvb.Add(Slices.Empty); // result
tvb.Add(Slices.Empty); // exception type
tvb.Add(Slices.Empty); // exception message
table.Set(tvb);
}
if (table.NumberOfEntries > _logHistoryMaxEntries)
{
var reader = table.SeekOneForwardFromPrefix(LogHistoryTable.Indexes[LogHistoryIndexSlice], Slices.BeforeAllKeys);
table.Delete(reader.Reader.Id);
}
}