public virtual unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { BlittableJsonReaderObject itemBlittable = null; var itemKey = GetItemId(); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) { if (items.ReadByKey(valueNameLowered, out TableValueReader reader)) { var ptr = reader.Read(2, out int size); itemBlittable = new BlittableJsonReaderObject(ptr, size, context); } itemBlittable = GetUpdatedValue(index, record, context, itemBlittable); // if returned null, means, there is nothing to update and we just wanted to delete the value if (itemBlittable == null) { items.DeleteByKey(valueNameLowered); result = GetResult(); return; } // here we get the item key again, in case it was changed (a new entity, etc) itemKey = GetItemId(); } using (Slice.From(context.Allocator, itemKey, out Slice valueName)) using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) { ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, itemBlittable); result = GetResult(); } }
public override void InitializeForNewBatch(ClusterOperationContext clusterContext, DocumentsOperationContext docsContext, IncludeDocumentsCommand includesCmd) { base.InitializeForNewBatch(clusterContext, docsContext, includesCmd); Fetcher = CreateFetcher(); Fetcher.Initialize(clusterContext, docsContext, Active); }
private bool InstallSnapshot(ClusterOperationContext context, CancellationToken token) { var txw = context.Transaction.InnerTransaction; var fileName = $"snapshot.{Guid.NewGuid():N}"; var filePath = context.Environment.Options.DataPager.Options.TempPath.Combine(fileName); using (var temp = new StreamsTempFile(filePath.FullPath, context.Environment)) using (var stream = temp.StartNewStream()) using (var remoteReader = _connection.CreateReaderToStream(stream)) { if (ReadSnapshot(remoteReader, context, txw, dryRun: true, token) == false) { return(false); } stream.Seek(0, SeekOrigin.Begin); using (var fileReader = new StreamSnapshotReader(stream)) { ReadSnapshot(fileReader, context, txw, dryRun: false, token); } } return(true); }
public unsafe Dictionary <string, long> Clean(ClusterOperationContext context, long index) { var affectedDatabases = new Dictionary <string, long>(); foreach (var tuple in ClusterTransactionsCleanup) { var database = tuple.Key; var upToCommandCount = tuple.Value - 1; var items = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.TransactionCommandsSchema, ClusterStateMachine.TransactionCommands); using (ClusterTransactionCommand.GetPrefix(context, database, out var prefixSlice)) { var deleted = items.DeleteByPrimaryKeyPrefix(prefixSlice, shouldAbort: (tvb) => { var value = tvb.Reader.Read((int)ClusterTransactionCommand.TransactionCommandsColumn.Key, out var size); var prevCommandsCount = Bits.SwapBytes(*(long *)(value + size - sizeof(long))); return(prevCommandsCount > upToCommandCount); }); if (deleted) { affectedDatabases.Add(database, tuple.Value); } } } return(affectedDatabases); }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { result = null; var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) { if (items.ReadByKey(valueNameLowered, out TableValueReader tvr) == false) { return; // nothing to do } var ptr = tvr.Read(2, out int size); var doc = new BlittableJsonReaderObject(ptr, size, context); var subscriptionState = JsonDeserializationClient.SubscriptionState(doc); items.DeleteByKey(valueNameLowered); if (string.IsNullOrEmpty(subscriptionState.SubscriptionName) == false) { itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, subscriptionState.SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out valueNameLowered)) { items.DeleteByKey(valueNameLowered); } using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionPrefix(context, DatabaseName, subscriptionState.SubscriptionId, out var prefix)) { var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState); using var _ = Slice.External(context.Allocator, prefix, out var prefixSlice); subscriptionStateTable.DeleteByPrimaryKeyPrefix(prefixSlice); } } } }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { var identitiesItems = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.IdentitiesSchema, ClusterStateMachine.Identities); var listResult = new List <long>(); foreach (var identity in Identities) { CompareExchangeCommandBase.GetKeyAndPrefixIndexSlices(context.Allocator, DatabaseName, identity, index, out var keyTuple, out var indexTuple); using (keyTuple.Scope) using (indexTuple.Scope) using (Slice.External(context.Allocator, keyTuple.Buffer.Ptr, keyTuple.Buffer.Length, out var keySlice)) using (Slice.External(context.Allocator, indexTuple.Buffer.Ptr, indexTuple.Buffer.Length, out var prefixIndexSlice)) { long value; if (identitiesItems.ReadByKey(keySlice, out var reader)) { value = GetValue(reader); value += 1; } else { value = 1; } UpdateTableRow(index, identitiesItems, value, keySlice, prefixIndexSlice); listResult.Add(value); } } result = listResult; }
public string Read(ClusterOperationContext context, string name) { var tree = context.Transaction.InnerTransaction.ReadTree("values"); var read = tree.Read(name); return(read?.Reader.ToStringValue()); }
public long FindFreeId(ClusterOperationContext context, long subscriptionId) { if (SubscriptionId.HasValue) { return(SubscriptionId.Value); } bool idTaken; do { idTaken = false; foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(DatabaseName))) { if (keyValue.Value.TryGet(nameof(SubscriptionState.SubscriptionId), out long id) == false) { continue; } if (id == subscriptionId) { subscriptionId--; // we don't care if this end up as a negative value, we need only to be unique idTaken = true; break; } } } while (idTaken); return(subscriptionId); }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { var identitiesItems = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.IdentitiesSchema, ClusterStateMachine.Identities); CompareExchangeCommandBase.GetKeyAndPrefixIndexSlices(context.Allocator, DatabaseName, Prefix, index, out var keyTuple, out var indexTuple); using (keyTuple.Scope) using (indexTuple.Scope) using (Slice.External(context.Allocator, keyTuple.Buffer.Ptr, keyTuple.Buffer.Length, out var keySlice)) using (Slice.External(context.Allocator, indexTuple.Buffer.Ptr, indexTuple.Buffer.Length, out var prefixIndexSlice)) { long value; if (identitiesItems.SeekOnePrimaryKeyPrefix(keySlice, out var entry)) { value = GetValue(entry); value++; } else { value = 1; } UpdateTableRow(index, identitiesItems, value, keySlice, prefixIndexSlice); result = value; } }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { result = null; var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) { if (items.ReadByKey(valueNameLowered, out TableValueReader tvr) == false) { return; // nothing to do } var ptr = tvr.Read(2, out int size); var doc = new BlittableJsonReaderObject(ptr, size, context); var subscriptionState = JsonDeserializationClient.SubscriptionState(doc); items.DeleteByKey(valueNameLowered); if (string.IsNullOrEmpty(subscriptionState.SubscriptionName) == false) { itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, subscriptionState.SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out valueNameLowered)) { items.DeleteByKey(valueNameLowered); } } } }
public override unsafe void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { result = null; var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName); using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered)) using (Slice.From(context.Allocator, itemKey, out Slice valueName)) { if (items.ReadByKey(valueNameLowered, out var tvr) == false) { throw new SubscriptionDoesNotExistException($"Cannot find subscription {SubscriptionName} @ {DatabaseName}"); } var ptr = tvr.Read(2, out int size); var doc = new BlittableJsonReaderObject(ptr, size, context); var subscriptionState = JsonDeserializationClient.SubscriptionState(doc); subscriptionState.Disabled = Disable; using (var obj = context.ReadObject(subscriptionState.ToJson(), "subscription")) { ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, obj); } } }
public virtual void Initialize(RachisConsensus parent, ClusterOperationContext context, ClusterChanges changes) { _parent = parent; ContextPoolForReadOnlyOperations = _parent.ContextPool; Changes = changes; Validator = InitializeValidator(); }
public string CollectLogs(ClusterOperationContext context, RavenServer server) { return ($"{Environment.NewLine}Log for server '{server.ServerStore.NodeTag}':" + $"{Environment.NewLine}Last notified Index '{server.ServerStore.Cluster.LastNotifiedIndex}':" + $"{Environment.NewLine}{context.ReadObject(server.ServerStore.GetLogDetails(context, max: int.MaxValue), "LogSummary/" + server.ServerStore.NodeTag)}" + $"{Environment.NewLine}{server.ServerStore.Engine.LogHistory.GetHistoryLogsAsString(context)}"); }
private unsafe void UpdateInternal(ClusterOperationContext context, string guid, string type, long index, long term, HistoryStatus status, object result, Exception exception) { var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice); TableValueReader reader; using (Slice.From(context.Allocator, guid, out var guidSlice)) { if (table.ReadByKey(guidSlice, out reader) == false) { return; } } if (TypeConverter.IsSupportedType(result) == false) { throw new RachisApplyException("We don't support type " + result.GetType().FullName + "."); } using (Slice.From(context.Allocator, guid, out var guidSlice)) using (Slice.From(context.Allocator, type, out var typeSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(guidSlice); tvb.Add(Bits.SwapBytes(index)); tvb.Add(Bits.SwapBytes(GetUniqueTicks(context.Transaction.InnerTransaction))); tvb.Add(*(long *)reader.Read((int)(LogHistoryColumn.Term), out _)); tvb.Add(term); tvb.Add(typeSlice); tvb.Add((byte)status); if (result == null) { tvb.Add(Slices.Empty); } else { var blittableResult = context.ReadObject(new DynamicJsonValue { ["Result"] = result }, "set-history-result"); tvb.Add(blittableResult.BasePointer, blittableResult.Size); } if (exception == null) { tvb.Add(Slices.Empty); tvb.Add(Slices.Empty); } else { var exceptionType = context.GetLazyString(exception.GetType().AssemblyQualifiedName); var exceptionMsg = context.GetLazyString(exception.ToString()); tvb.Add(exceptionType.Buffer, exceptionType.Size); tvb.Add(exceptionMsg.Buffer, exceptionMsg.Size); } table.Set(tvb); } }
public IEnumerable <DynamicJsonValue> GetHistoryLogs(ClusterOperationContext context) { var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice); foreach (var entryHolder in table.SeekForwardFrom(LogHistoryTable.FixedSizeIndexes[LogHistoryDateTimeSlice], 0, 0)) { yield return(ReadHistoryLog(context, entryHolder)); } }
public static unsafe void Put(ClusterOperationContext context, Slice keySlice, long ticks) { var ticksBigEndian = Bits.SwapBytes(ticks); using (Slice.External(context.Allocator, (byte *)&ticksBigEndian, sizeof(long), out Slice ticksSlice)) { var tree = context.Transaction.InnerTransaction.ReadTree(CompareExchangeByExpiration); tree.MultiAdd(ticksSlice, keySlice); } }
protected override void Apply(ClusterOperationContext context, BlittableJsonReaderObject cmd, long index, Leader leader, ServerStore serverStore) { Assert.True(cmd.TryGet(nameof(TestCommand.Name), out string name)); Assert.True(cmd.TryGet(nameof(TestCommand.Value), out int val)); var tree = context.Transaction.InnerTransaction.CreateTree("values"); var current = tree.Read(name)?.Reader.ToStringValue(); tree.Add(name, current + val); }
public CompareExchangeResult Execute(ClusterOperationContext context, Table items, long index) { var result = ExecuteInternal(context, items, index); context.Transaction.AddAfterCommitNotification(new CompareExchangeChange { Database = Database }); return(result); }
public override void Execute(ClusterOperationContext context, Table items, long index, RawDatabaseRecord record, RachisState state, out object result) { base.Execute(context, items, index, record, state, out result); if (IsLegacyCommand()) { return; } ExecuteAcknowledgeSubscriptionBatch(context, index); }
public string GetHistoryLogsAsString(ClusterOperationContext context) { var sb = new StringBuilder(); foreach (var entry in GetHistoryLogs(context)) { sb.AppendLine(context.ReadObject(entry, "raft-command-history").ToString()); } return(sb.ToString()); }
private void RequestAllEntries(ClusterOperationContext context, RemoteConnection connection, string message) { connection.Send(context, new LogLengthNegotiationResponse { Status = LogLengthNegotiationResponse.ResponseStatus.Acceptable, Message = message, CurrentTerm = _term, LastLogIndex = 0 }); }
private unsafe void ExecuteAcknowledgeSubscriptionBatch(ClusterOperationContext context, long index) { if (SubscriptionId == default) { throw new RachisApplyException( $"'{nameof(SubscriptionId)}' is missing in '{nameof(AcknowledgeSubscriptionBatchCommand)}'."); } if (DatabaseName == default) { throw new RachisApplyException($"'{nameof(DatabaseName)}' is missing in '{nameof(AcknowledgeSubscriptionBatchCommand)}'."); } if (BatchId == null) { throw new RachisApplyException($"'{nameof(BatchId)}' is missing in '{nameof(AcknowledgeSubscriptionBatchCommand)}'."); } var subscriptionStateTable = context.Transaction.InnerTransaction.OpenTable(ClusterStateMachine.SubscriptionStateSchema, ClusterStateMachine.SubscriptionState); var bigEndBatchId = Bits.SwapBytes(BatchId ?? 0); using var _ = Slice.External(context.Allocator, (byte *)&bigEndBatchId, sizeof(long), out var batchIdSlice); subscriptionStateTable.DeleteForwardFrom(ClusterStateMachine.SubscriptionStateSchema.Indexes[ClusterStateMachine.SubscriptionStateByBatchIdSlice], batchIdSlice, false, long.MaxValue, shouldAbort: tvh => { var recordBatchId = Bits.SwapBytes(*(long *)tvh.Reader.Read((int)ClusterStateMachine.SubscriptionStateTable.BatchId, out var size)); return(recordBatchId != BatchId); }); if (DocumentsToResend == null) { return; } foreach (var r in DocumentsToResend) { using (SubscriptionConnectionsState.GetDatabaseAndSubscriptionAndDocumentKey(context, DatabaseName, SubscriptionId, r.DocumentId, out var key)) using (subscriptionStateTable.Allocate(out var tvb)) { using var __ = Slice.External(context.Allocator, key, out var keySlice); using var ___ = Slice.From(context.Allocator, r.ChangeVector, out var changeVectorSlice); tvb.Add(keySlice); tvb.Add(changeVectorSlice); tvb.Add(SwappedNonExistentBatch); // batch id subscriptionStateTable.Set(tvb); } } }
public bool ContainsCommandId(ClusterOperationContext context, string guid) { var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice); if (table == null) { return(false); } using (Slice.From(context.Allocator, guid, out var guidSlice)) { return(table.VerifyKeyExists(guidSlice)); } }
public long Apply(ClusterOperationContext context, long uptoInclusive, Leader leader, ServerStore serverStore, Stopwatch duration) { Debug.Assert(context.Transaction != null); var lastAppliedIndex = _parent.GetLastCommitIndex(context); var maxTimeAllowedToWaitForApply = _parent.Timeout.TimeoutPeriod / 4; for (var index = lastAppliedIndex + 1; index <= uptoInclusive; index++) { var cmd = _parent.GetEntry(context, index, out RachisEntryFlags flags); if (cmd == null || flags == RachisEntryFlags.Invalid) { throw new InvalidOperationException("Expected to apply entry " + index + " but it isn't stored"); } lastAppliedIndex = index; if (flags != RachisEntryFlags.StateMachineCommand) { _parent.LogHistory.UpdateHistoryLog(context, index, _parent.CurrentTerm, cmd, null, null); var currentIndex = index; context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += t => { if (t is LowLevelTransaction llt && llt.Committed) { serverStore.Cluster.NotifyAndSetCompleted(currentIndex); } }; continue; } Apply(context, cmd, index, leader, serverStore); if (duration.ElapsedMilliseconds >= maxTimeAllowedToWaitForApply) { // we don't want to spend so much time applying commands that we will time out the leader // so we time this from the follower perspective and abort after applying a single command // or 25% of the time has already passed break; } } var term = _parent.GetTermForKnownExisting(context, lastAppliedIndex); _parent.SetLastCommitIndex(context, lastAppliedIndex, term); return(lastAppliedIndex); }
public Dictionary <string, long> Clean(ClusterOperationContext context, long index) { var affectedDatabases = new Dictionary <string, long>(); foreach (var tuple in ClusterTransactionsCleanup) { var database = tuple.Key; var upToCommandCount = tuple.Value - 1; if (ClusterTransactionCommand.DeleteCommands(context, database, upToCommandCount)) { affectedDatabases.Add(database, tuple.Value); } } return(affectedDatabases); }
private static void CleanExpired(ClusterOperationContext context, Dictionary <Slice, List <Slice> > expired) { if (expired.Count == 0) { return; } var expirationTree = context.Transaction.InnerTransaction.ReadTree(CompareExchangeByExpiration); foreach (var pair in expired) { foreach (var ids in pair.Value) { expirationTree.MultiDelete(pair.Key, ids); } } }
private HandleVoteResult ShouldGrantVote(ClusterOperationContext context, long lastIndex, RequestVote rv) { var result = new HandleVoteResult(); var lastLogIndexUnderWriteLock = _engine.GetLastEntryIndex(context); var lastLogTermUnderWriteLock = _engine.GetTermFor(context, lastLogIndexUnderWriteLock); if (lastLogIndexUnderWriteLock != lastIndex) { result.DeclineVote = true; result.DeclineReason = "Log was changed"; return(result); } if (lastLogTermUnderWriteLock > rv.LastLogTerm) { result.DeclineVote = true; result.DeclineReason = $"My last log term {lastLogTermUnderWriteLock}, is higher than yours {rv.LastLogTerm}."; return(result); } if (lastLogIndexUnderWriteLock > rv.LastLogIndex) { result.DeclineVote = true; result.DeclineReason = $"Vote declined because my last log index {lastLogIndexUnderWriteLock} is more up to date than yours {rv.LastLogIndex}"; return(result); } var(whoGotMyVoteIn, votedTerm) = _engine.GetWhoGotMyVoteIn(context, rv.Term); result.VotedTerm = votedTerm; if (whoGotMyVoteIn != null && whoGotMyVoteIn != rv.Source) { result.DeclineVote = true; result.DeclineReason = $"Already voted in {rv.LastLogTerm}, for {whoGotMyVoteIn}"; return(result); } if (votedTerm >= rv.Term) { result.DeclineVote = true; result.DeclineReason = $"Already voted in {rv.LastLogTerm}, for another node in higher term: {votedTerm}"; return(result); } return(result); }
public void UpdateHistoryLog(ClusterOperationContext context, long index, long term, BlittableJsonReaderObject cmd, object result, Exception exception) { var guid = GetGuidFromCommand(cmd); if (guid == null) // shouldn't happened in new cluster version! { return; } if (guid == RaftIdGenerator.DontCareId) { return; } var type = GetTypeFromCommand(cmd); UpdateInternal(context, guid, type, index, term, HistoryStatus.Committed, result, exception); }
private bool CanHandleLogDivergence(ClusterOperationContext context, LogLengthNegotiation negotiation, ref long midpointIndex, ref long midpointTerm, ref long minIndex, ref long maxIndex) { if (_engine.Log.IsInfoEnabled) { _engine.Log.Info( $"{ToString()}: Our appended entries has been diverged, same index with different terms. " + $"My index/term {midpointIndex}/{midpointTerm}, while yours is {negotiation.PrevLogIndex}/{negotiation.PrevLogTerm}."); } using (context.OpenReadTransaction()) { do { // try to find any log in the previous term midpointIndex--; midpointTerm = _engine.GetTermFor(context, midpointIndex) ?? 0; } while (midpointTerm >= negotiation.PrevLogTerm && midpointIndex > 0); if (midpointTerm == 0 || midpointIndex == 0) { return(false); } // start the binary search again with those boundaries minIndex = Math.Min(midpointIndex, _engine.GetFirstEntryIndex(context)); maxIndex = midpointIndex; midpointIndex = (minIndex + maxIndex) / 2; midpointTerm = _engine.GetTermForKnownExisting(context, midpointIndex); if (maxIndex < minIndex) { if (_engine.Log.IsInfoEnabled) { _engine.Log.Info($"{ToString()}: Got minIndex: {minIndex} bigger than maxIndex: {maxIndex} will request the entire snapshot. " + $"midpointIndex: {midpointIndex}, midpointTerm: {midpointTerm}."); } Debug.Assert(false, "This is a safeguard against any potential bug here, so in worst case we request the entire snapshot"); return(false); } } return(true); }
public void InsertHistoryLog(ClusterOperationContext context, long index, long term, BlittableJsonReaderObject cmd) { if (HasHistoryLog(context, cmd, out _, out _, out _)) { return; } var guid = GetGuidFromCommand(cmd); if (guid == null) // shouldn't happened in new cluster version! { return; } if (guid == RaftIdGenerator.DontCareId) { return; } var table = context.Transaction.InnerTransaction.OpenTable(LogHistoryTable, LogHistorySlice); var type = GetTypeFromCommand(cmd); using (Slice.From(context.Allocator, guid, out var guidSlice)) using (Slice.From(context.Allocator, type, out var typeSlice)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(guidSlice); tvb.Add(Bits.SwapBytes(index)); tvb.Add(Bits.SwapBytes(GetUniqueTicks(context.Transaction.InnerTransaction))); tvb.Add(term); tvb.Add(0L); tvb.Add(typeSlice); tvb.Add((byte)HistoryStatus.Appended); tvb.Add(Slices.Empty); // result tvb.Add(Slices.Empty); // exception type tvb.Add(Slices.Empty); // exception message table.Set(tvb); } if (table.NumberOfEntries > _logHistoryMaxEntries) { var reader = table.SeekOneForwardFromPrefix(LogHistoryTable.Indexes[LogHistoryIndexSlice], Slices.BeforeAllKeys); table.Delete(reader.Reader.Id); } }