public override async Task AcknowledgeBatch(long batchId) { ItemsToRemoveFromResend.Clear(); //pick up docs that weren't sent due to having been processed by this connection and add them to resend using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docContext)) using (docContext.OpenReadTransaction()) { for (var index = BatchItems.Count - 1; index >= 0; index--) { var doc = BatchItems[index]; var document = Database.DocumentsStorage.GetDocumentOrTombstone(docContext, doc.DocumentId, throwOnConflict: false); if (ShouldAddToResendTable(document, doc.ChangeVector) == false) { BatchItems.RemoveAt(index); } } } await SubscriptionConnectionsState.AcknowledgeBatch(Connection, batchId, BatchItems); if (BatchItems?.Count > 0) { SubscriptionConnectionsState.NotifyHasMoreDocs(); BatchItems.Clear(); } }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { result = null; var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) { if (items.ReadByKey(valueNameLowered, out TableValueReader tvr) == false) { return; // nothing to do } var ptr = tvr.Read(2, out int size); var doc = new BlittableJsonReaderObject(ptr, size, context); var subscriptionState = JsonDeserializationClient.SubscriptionState(doc); items.DeleteByKey(valueNameLowered); if (string.IsNullOrEmpty(subscriptionState.SubscriptionName) == false) { itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, subscriptionState.SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out valueNameLowered)) { items.DeleteByKey(valueNameLowered); } using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionPrefix(context, DatabaseName, subscriptionState.SubscriptionId, out var prefix)) { var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState); using var _ = Slice.External(context.Allocator, prefix, out var prefixSlice); subscriptionStateTable.DeleteByPrimaryKeyPrefix(prefixSlice); } } } }
private unsafe void ExecuteAcknowledgeSubscriptionBatch(ClusterOperationContext context, long index) { if (SubscriptionId == default) { throw new RachisApplyException( $"'{nameof(SubscriptionId)}' is missing in '{nameof(AcknowledgeSubscriptionBatchCommand)}'."); } if (DatabaseName == default) { throw new RachisApplyException($"'{nameof(DatabaseName)}' is missing in '{nameof(AcknowledgeSubscriptionBatchCommand)}'."); } if (BatchId == null) { throw new RachisApplyException($"'{nameof(BatchId)}' is missing in '{nameof(AcknowledgeSubscriptionBatchCommand)}'."); } var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState); var bigEndBatchId = Bits.SwapBytes(BatchId ?? 0); using var _ = Slice.External(context.Allocator, (byte *)&bigEndBatchId, sizeof(long), out var batchIdSlice); subscriptionStateTable.DeleteForwardFrom(ClusterStateMachine.SubscriptionStateSchema.Indexes[ClusterStateMachine.SubscriptionStateByBatchIdSlice], batchIdSlice, false, long.MaxValue, shouldAbort: tvh => { var recordBatchId = Bits.SwapBytes(*(long *)tvh.Reader.Read((int)ClusterStateMachine.SubscriptionStateTable.BatchId, out var size)); return(recordBatchId != BatchId); }); if (DocumentsToResend == null) { return; } foreach (var r in DocumentsToResend) { using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndDocumentKey(context, DatabaseName, SubscriptionId, r.DocumentId, out var key)) using (subscriptionStateTable.Allocate(out var tvb)) { using var __ = Slice.External(context.Allocator, key, out var keySlice); using var ___ = Slice.From(context.Allocator, r.ChangeVector, out var changeVectorSlice); tvb.Add(keySlice); tvb.Add(changeVectorSlice); tvb.Add(SwappedNonExistentBatch); // batch id subscriptionStateTable.Set(tvb); } } }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { long i = 1; var originalName = SubscriptionName; var tryToSetName = true; result = null; var subscriptionId = SubscriptionId ?? index; SubscriptionName = string.IsNullOrEmpty(SubscriptionName) ? subscriptionId.ToString() : SubscriptionName; var baseName = SubscriptionName; if (SubscriptionName.Length > DocumentIdWorker.MaxIdSize) { throw new SubscriptionNameException($"Subscription Name is too long, must be at most {DocumentIdWorker.MaxIdSize} bytes"); } while (tryToSetName) { var subscriptionItemName = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName); using (Slice.From(context.Allocator, subscriptionItemName, out Slice valueName)) using (Slice.From(context.Allocator, subscriptionItemName.ToLowerInvariant(), out Slice valueNameLowered)) { if (items.ReadByKey(valueNameLowered, out TableValueReader tvr)) { var ptr = tvr.Read(2, out int size); var doc = new BlittableJsonReaderObject(ptr, size, context); var existingSubscriptionState = JsonDeserializationClient.SubscriptionState(doc); if (SubscriptionId != existingSubscriptionState.SubscriptionId) { if (string.IsNullOrEmpty(originalName)) { SubscriptionName = $"{baseName}.{i}"; i++; continue; } throw new RachisApplyException("A subscription could not be modified because the name '" + subscriptionItemName + "' is already in use in a subscription with different Id."); } if (string.IsNullOrEmpty(InitialChangeVector) == false && InitialChangeVector == nameof(Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange)) { InitialChangeVector = existingSubscriptionState.ChangeVectorForNextBatchStartingPoint; } else { AssertValidChangeVector(); if (InitialChangeVector != existingSubscriptionState.ChangeVectorForNextBatchStartingPoint) { // modified by the admin var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState); using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionPrefix(context, DatabaseName, subscriptionId, out var prefix)) { using var _ = Slice.External(context.Allocator, prefix, out var prefixSlice); subscriptionStateTable.DeleteByPrimaryKeyPrefix(prefixSlice); } } } } else { AssertValidChangeVector(); } using (var receivedSubscriptionState = context.ReadObject(new SubscriptionState { Query = Query, ChangeVectorForNextBatchStartingPoint = InitialChangeVector, SubscriptionId = subscriptionId, SubscriptionName = SubscriptionName, LastBatchAckTime = null, Disabled = Disabled, MentorNode = MentorNode, LastClientConnectionTime = null }.ToJson(), SubscriptionName)) { ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, receivedSubscriptionState); } tryToSetName = false; } } }
public DocumentSubscriptionFetcher(DocumentDatabase database, SubscriptionConnectionsState subscriptionConnectionsState, string collection) : base(database, subscriptionConnectionsState, collection) { }
public override void InitializeProcessor() { SubscriptionConnectionsState = SubscriptionConnectionsState.CreateDummyState(Database.DocumentsStorage, SubscriptionState); }
protected override bool ShouldSend(Document item, out string reason, out Exception exception, out Document result) { exception = null; reason = null; result = item; if (Fetcher.FetchingFrom == SubscriptionFetcher.FetchingOrigin.Storage) { var conflictStatus = ChangeVectorUtils.GetConflictStatus( remoteAsString: item.ChangeVector, localAsString: SubscriptionState.ChangeVectorForNextBatchStartingPoint); if (conflictStatus == ConflictStatus.AlreadyMerged) { reason = $"{item.Id} is already merged"; return(false); } if (SubscriptionConnectionsState.IsDocumentInActiveBatch(ClusterContext, item.Id, Active)) { reason = $"{item.Id} exists in an active batch"; return(false); } } if (Fetcher.FetchingFrom == SubscriptionFetcher.FetchingOrigin.Resend) { var current = Database.DocumentsStorage.GetDocumentOrTombstone(DocsContext, item.Id, throwOnConflict: false); if (ShouldFetchFromResend(item.Id, current, item.ChangeVector) == false) { item.ChangeVector = string.Empty; reason = $"Skip {item.Id} from resend"; return(false); } Debug.Assert(current.Document != null, "Document does not exist"); result.Id = current.Document.Id; // use proper casing result.Data = current.Document.Data; result.Etag = current.Document.Etag; result.ChangeVector = current.Document.ChangeVector; } if (Patch == null) { return(true); } try { InitializeScript(); var match = Patch.MatchCriteria(Run, DocsContext, item, ProjectionMetadataModifier.Instance, ref result.Data); if (match == false) { if (Fetcher.FetchingFrom == SubscriptionFetcher.FetchingOrigin.Resend) { ItemsToRemoveFromResend.Add(item.Id); } result.Data = null; reason = $"{item.Id} filtered out by criteria"; return(false); } return(true); } catch (Exception ex) { exception = ex; reason = $"Criteria script threw exception for document id {item.Id}"; return(false); } }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { result = null; var shouldUpdateChangeVector = true; var subscriptionName = SubscriptionName; if (string.IsNullOrEmpty(subscriptionName)) { subscriptionName = SubscriptionId.ToString(); } //insert all docs to voron table. If exists, then batchId will be replaced var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState); var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, subscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) using (Slice.From(context.Allocator, itemKey, out Slice valueName)) { if (items.ReadByKey(valueNameLowered, out var tvr) == false) { throw new RachisApplyException($"Cannot find subscription {subscriptionName} @ {DatabaseName}"); } var ptr = tvr.Read(2, out int size); var existingValue = new BlittableJsonReaderObject(ptr, size, context); if (existingValue == null) { throw new SubscriptionDoesNotExistException($"Subscription with name '{subscriptionName}' does not exist in database '{DatabaseName}'"); } var subscriptionState = JsonDeserializationClient.SubscriptionState(existingValue); var topology = record.Topology; var lastResponsibleNode = AcknowledgeSubscriptionBatchCommand.GetLastResponsibleNode(HasHighlyAvailableTasks, topology, NodeTag); var appropriateNode = topology.WhoseTaskIsIt(RachisState.Follower, subscriptionState, lastResponsibleNode); if (appropriateNode == null && record.DeletionInProgress.ContainsKey(NodeTag)) { throw new DatabaseDoesNotExistException($"Stopping subscription '{subscriptionName}' on node {NodeTag}, because database '{DatabaseName}' is being deleted."); } if (appropriateNode != NodeTag) { throw new SubscriptionDoesNotBelongToNodeException( $"Cannot apply {nameof(AcknowledgeSubscriptionBatchCommand)} for subscription '{subscriptionName}' with id '{SubscriptionId}', on database '{DatabaseName}', on node '{NodeTag}'," + $" because the subscription task belongs to '{appropriateNode ?? "N/A"}'.") { AppropriateNode = appropriateNode }; } if (CurrentChangeVector == nameof(Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange)) { context.ReadObject(existingValue, subscriptionName); shouldUpdateChangeVector = false; } if (subscriptionState.ChangeVectorForNextBatchStartingPoint != PreviouslyRecordedChangeVector) { throw new SubscriptionChangeVectorUpdateConcurrencyException($"Can't record subscription with name '{subscriptionName}' due to inconsistency in change vector progress. Probably there was an admin intervention that changed the change vector value. Stored value: {subscriptionState.ChangeVectorForNextBatchStartingPoint}, received value: {PreviouslyRecordedChangeVector}"); } if (shouldUpdateChangeVector) { subscriptionState.ChangeVectorForNextBatchStartingPoint = ChangeVectorUtils.MergeVectors(CurrentChangeVector, subscriptionState.ChangeVectorForNextBatchStartingPoint); subscriptionState.NodeTag = NodeTag; using (var obj = context.ReadObject(subscriptionState.ToJson(), "subscription")) { ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, obj); } } } foreach (var deletedId in Deleted) { using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndDocumentKey(context, DatabaseName, SubscriptionId, deletedId, out var key)) { using var _ = Slice.External(context.Allocator, key, out var keySlice); subscriptionStateTable.DeleteByKey(keySlice); } } foreach (var documentRecord in Documents) { using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndDocumentKey(context, DatabaseName, SubscriptionId, documentRecord.DocumentId, out var key)) using (subscriptionStateTable.Allocate(out var tvb)) { using var _ = Slice.External(context.Allocator, key, out var keySlice); using var __ = Slice.From(context.Allocator, documentRecord.ChangeVector, out var changeVectorSlice); tvb.Add(keySlice); tvb.Add(changeVectorSlice); tvb.Add(Bits.SwapBytes(index)); // batch id subscriptionStateTable.Set(tvb); } } foreach (var revisionRecord in Revisions) { using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndRevisionKey(context, DatabaseName, SubscriptionId, revisionRecord.Current, out var key)) using (subscriptionStateTable.Allocate(out var tvb)) { using var _ = Slice.External(context.Allocator, key, out var keySlice); using var __ = Slice.From(context.Allocator, revisionRecord.Previous ?? string.Empty, out var changeVectorSlice); tvb.Add(keySlice); tvb.Add(changeVectorSlice); //prev change vector tvb.Add(Bits.SwapBytes(index)); // batch id subscriptionStateTable.Set(tvb); } } }