public virtual unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, RachisState state, out object result)
{
BlittableJsonReaderObject itemBlittable = null;
var itemKey = GetItemId();
using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered))
{
if (items.ReadByKey(valueNameLowered, out TableValueReader reader))
{
var ptr = reader.Read(2, out int size);
itemBlittable = new BlittableJsonReaderObject(ptr, size, context);
}
itemBlittable = GetUpdatedValue(index, record, context, itemBlittable, state);
// if returned null, means, there is nothing to update and we just wanted to delete the value
if (itemBlittable == null)
{
items.DeleteByKey(valueNameLowered);
result = GetResult();
return;
}
// here we get the item key again, in case it was changed (a new entity, etc)
itemKey = GetItemId();
}
using (Slice.From(context.Allocator, itemKey, out Slice valueName))
using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered))
{
ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, itemBlittable);
result = GetResult();
}
}
public long FindFreeId(ClusterOperationContext context, long subscriptionId)
{
if (SubscriptionId.HasValue)
{
return(SubscriptionId.Value);
}
bool idTaken;
do
{
idTaken = false;
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context,
SubscriptionState.SubscriptionPrefix(DatabaseName)))
{
if (keyValue.Value.TryGet(nameof(SubscriptionState.SubscriptionId), out long id) == false)
{
continue;
}
if (id == subscriptionId)
{
subscriptionId--; // we don't care if this end up as a negative value, we need only to be unique
idTaken = true;
break;
}
}
} while (idTaken);
return(subscriptionId);
}
public void RecomputeChangesStatesAsExpected()
{
var clusterState = new ClusterStateMachine();
var cfg = new ClusterStateRecomputeConfiguration()
{
// Force recompute to run
RecomputeFrequency = TimeSpan.Zero,
};
var nowUtc = _clock.UtcNow;
// We want to test all possible state machine transitions. To do so, we generate a very specific instance
// of cluster state meant to transition the way we expect instead of actually simulating each possible
// branch.
var n1 = MachineLocation.Create("node2", 0);
MachineId n1Id;
(clusterState, n1Id) = clusterState.ForceRegisterMachineWithState(n1, nowUtc, MachineState.DeadUnavailable);
var n2 = MachineLocation.Create("node3", 0);
MachineId n2Id;
(clusterState, n2Id) = clusterState.ForceRegisterMachineWithState(n2, nowUtc, MachineState.DeadExpired);
var n3 = MachineLocation.Create("node4", 0);
MachineId n3Id;
(clusterState, n3Id) = clusterState.ForceRegisterMachineWithState(n3, nowUtc - cfg.ActiveToDeadExpiredInterval, MachineState.Open);
var n4 = MachineLocation.Create("node5", 0);
MachineId n4Id;
(clusterState, n4Id) = clusterState.ForceRegisterMachineWithState(n4, nowUtc - cfg.ActiveToClosedInterval, MachineState.Open);
var n5 = MachineLocation.Create("node6", 0);
MachineId n5Id;
(clusterState, n5Id) = clusterState.ForceRegisterMachineWithState(n5, nowUtc, MachineState.Open);
var n6 = MachineLocation.Create("node7", 0);
MachineId n6Id;
(clusterState, n6Id) = clusterState.ForceRegisterMachineWithState(n6, nowUtc - cfg.ClosedToDeadExpiredInterval, MachineState.Closed);
var n7 = MachineLocation.Create("node8", 0);
MachineId n7Id;
(clusterState, n7Id) = clusterState.ForceRegisterMachineWithState(n7, nowUtc, MachineState.Closed);
clusterState = clusterState.Recompute(cfg, nowUtc);
clusterState.GetStatus(n1Id).ThrowIfFailure().State.Should().Be(MachineState.DeadUnavailable);
clusterState.GetStatus(n2Id).ThrowIfFailure().State.Should().Be(MachineState.DeadExpired);
clusterState.GetStatus(n3Id).ThrowIfFailure().State.Should().Be(MachineState.DeadExpired);
clusterState.GetStatus(n4Id).ThrowIfFailure().State.Should().Be(MachineState.Closed);
clusterState.GetStatus(n5Id).ThrowIfFailure().State.Should().Be(MachineState.Open);
clusterState.GetStatus(n6Id).ThrowIfFailure().State.Should().Be(MachineState.DeadExpired);
clusterState.GetStatus(n7Id).ThrowIfFailure().State.Should().Be(MachineState.Closed);
}
public void Test_InvalidTopology()
{
StateChanged <InstanceState> stateChanged = null;
var observer = new MockStateObserver(s => stateChanged = s, Assert.IsNull);
InstanceState localState;
ClusterException cex = null;
ClusterStateMachine machine = CreateClusterStateMachine(observer, NodeRole.Primary, out localState,
ex => cex = ex);
var remoteState = new NodeState {
Role = NodeRole.Primary, Status = NodeStatus.Connecting
};
machine.RaiseEvent(localState, machine.PartnerStatusReceived, remoteState);
Assert.AreEqual(ClusterFailureReason.InvalidTopology, cex.Reason);
Assert.AreEqual("Stopped", stateChanged.Previous.Name);
Assert.AreEqual("Final", stateChanged.Current.Name);
machine = CreateClusterStateMachine(observer, NodeRole.Backup, out localState, ex => cex = ex);
remoteState = new NodeState {
Role = NodeRole.Backup, Status = NodeStatus.Connecting
};
machine.RaiseEvent(localState, machine.PartnerStatusReceived, remoteState);
Assert.AreEqual(ClusterFailureReason.InvalidTopology, cex.Reason);
Assert.AreEqual("Stopped", stateChanged.Previous.Name);
Assert.AreEqual("Final", stateChanged.Current.Name);
}
public void Test_SplitBrain_Backup()
{
StateChanged <InstanceState> stateChanged = null;
var observer = new MockStateObserver(s => stateChanged = s, Assert.IsNull);
InstanceState localState;
ClusterException cex = null;
ClusterStateMachine machine = CreateClusterStateMachine(observer, NodeRole.Backup, out localState,
ex => cex = ex);
var newState = new InstanceState
{
CurrentState = machine.Active,
Role = NodeRole.Backup,
Status = NodeStatus.Active
};
machine.TransitionToState(newState, machine.Active);
var remoteState = new NodeState {
Role = NodeRole.Primary, Status = NodeStatus.Active
};
machine.RaiseEvent(newState, machine.PartnerStatusReceived, remoteState);
Assert.AreEqual(ClusterFailureReason.SplitBrain, cex.Reason);
Assert.AreEqual("Stopped", stateChanged.Previous.Name);
Assert.AreEqual("Final", stateChanged.Current.Name);
}
public void RavenDB_13724()
{
var folder = NewDataPath(forceCreateDir: true, prefix: Guid.NewGuid().ToString());
DoNotReuseServer();
var zipPath = new PathSetting("SchemaUpgrade/Issues/SystemVersion/RavenDB_13724.zip");
Assert.True(File.Exists(zipPath.FullPath));
ZipFile.ExtractToDirectory(zipPath.FullPath, folder);
using (var server = GetNewServer(new ServerCreationOptions {
DeletePrevious = false, RunInMemory = false, DataDirectory = folder, RegisterForDisposal = false
}))
using (server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var db = server.ServerStore.Cluster.GetDatabaseNames(context).Single();
var ids = new HashSet <long>();
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(db)))
{
var state = JsonDeserializationClient.SubscriptionState(keyValue.Value);
Assert.True(ids.Add(state.SubscriptionId));
}
}
}
public void RegisterMachineTransitionTests()
{
var clusterState = new ClusterStateMachine();
var nowUtc = _clock.UtcNow;
// During transition, all machines will be added forcefully without regard for the consistency of the data
// structure
clusterState = clusterState.ForceRegisterMachine(new MachineId(3), MachineLocation.Create("A", 0), nowUtc);
clusterState.NextMachineId.Should().Be(4);
clusterState = clusterState.ForceRegisterMachine(new MachineId(8), MachineLocation.Create("B", 0), nowUtc);
clusterState.NextMachineId.Should().Be(9);
clusterState = clusterState.ForceRegisterMachine(new MachineId(16), MachineLocation.Create("C", 0), nowUtc);
clusterState.NextMachineId.Should().Be(17);
clusterState = clusterState.ForceRegisterMachine(new MachineId(20), MachineLocation.Create("D", 0), nowUtc);
clusterState.NextMachineId.Should().Be(21);
clusterState = clusterState.ForceRegisterMachine(new MachineId(23), MachineLocation.Create("D", 0), nowUtc);
clusterState.NextMachineId.Should().Be(24);
// After transition, adding proceeds as usual, by appending to the end basically
MachineId n1Id;
(clusterState, n1Id) = clusterState.RegisterMachine(MachineLocation.Create("Machine Gets Added After Transition", 0), nowUtc);
n1Id.Index.Should().Be(24);
clusterState.NextMachineId.Should().Be(25);
}
public override unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, RachisState state, out object result)
{
result = null;
var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName);
using (Slice.From(context.Allocator, itemKey.ToLowerInvariant(), out Slice valueNameLowered))
using (Slice.From(context.Allocator, itemKey, out Slice valueName))
{
if (items.ReadByKey(valueNameLowered, out var tvr) == false)
{
throw new RachisApplyException($"Cannot find subscription {index}");
}
var ptr = tvr.Read(2, out int size);
var doc = new BlittableJsonReaderObject(ptr, size, context);
var subscriptionState = JsonDeserializationClient.SubscriptionState(doc);
subscriptionState.Disabled = Disable;
using (var obj = context.ReadObject(subscriptionState.ToJson(), "subscription"))
{
ClusterStateMachine.UpdateValue(index, items, valueNameLowered, valueName, obj);
}
}
}
public override unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, bool isPassive, out object result)
{
result = null;
var subscriptionId = SubscriptionId ?? index;
SubscriptionName = string.IsNullOrEmpty(SubscriptionName) ? subscriptionId.ToString() : SubscriptionName;
var receivedSubscriptionState = context.ReadObject(new SubscriptionState
{
Query = Query,
ChangeVectorForNextBatchStartingPoint = InitialChangeVector,
SubscriptionId = subscriptionId,
SubscriptionName = SubscriptionName,
LastTimeServerMadeProgressWithDocuments = DateTime.UtcNow,
Disabled = Disabled,
LastClientConnectionTime = DateTime.Now
}.ToJson(), SubscriptionName);
BlittableJsonReaderObject modifiedSubscriptionState = null;
try
{
string subscriptionItemName = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName);
using (Slice.From(context.Allocator, subscriptionItemName, out Slice valueName))
using (Slice.From(context.Allocator, subscriptionItemName.ToLowerInvariant(), out Slice valueNameLowered))
{
if (items.ReadByKey(valueNameLowered, out TableValueReader tvr))
{
var ptr = tvr.Read(2, out int size);
var doc = new BlittableJsonReaderObject(ptr, size, context);
var existingSubscriptionState = JsonDeserializationClient.SubscriptionState(doc);
if (SubscriptionId != existingSubscriptionState.SubscriptionId)
{
throw new InvalidOperationException("A subscription could not be modified because the name '" + subscriptionItemName +
"' is already in use in a subscription with different Id.");
}
if (Enum.TryParse(InitialChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorState) && changeVectorState == Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange)
{
if (receivedSubscriptionState.Modifications == null)
{
receivedSubscriptionState.Modifications = new DynamicJsonValue();
}
receivedSubscriptionState.Modifications[nameof(SubscriptionState.ChangeVectorForNextBatchStartingPoint)] = existingSubscriptionState.ChangeVectorForNextBatchStartingPoint;
modifiedSubscriptionState = context.ReadObject(receivedSubscriptionState, SubscriptionName);
}
}
ClusterStateMachine.UpdateValue(subscriptionId, items, valueNameLowered, valueName, modifiedSubscriptionState ?? receivedSubscriptionState);
}
}
finally
{
receivedSubscriptionState.Dispose();
modifiedSubscriptionState?.Dispose();
}
}
public long GetAllSubscriptionsCount()
{
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
return(ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(_db.Name))
.Count());
}
}
public void Test_Start_When_Initial()
{
StateChanged <InstanceState> stateChanged = null;
var observer = new MockStateObserver(s => stateChanged = s, Assert.IsNull);
InstanceState localState;
ClusterStateMachine machine = CreateClusterStateMachine(observer, NodeRole.Primary, out localState);
Assert.AreEqual("Initial", stateChanged.Previous.Name);
Assert.AreEqual("Connecting", stateChanged.Current.Name);
}
public override unsafe void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, RachisState state, out object result)
{
result = null;
var subscriptionId = SubscriptionId ?? index;
SubscriptionName = string.IsNullOrEmpty(SubscriptionName) ? subscriptionId.ToString() : SubscriptionName;
var subscriptionItemName = SubscriptionState.GenerateSubscriptionItemKeyName(DatabaseName, SubscriptionName);
using (Slice.From(context.Allocator, subscriptionItemName, out Slice valueName))
using (Slice.From(context.Allocator, subscriptionItemName.ToLowerInvariant(), out Slice valueNameLowered))
{
if (items.ReadByKey(valueNameLowered, out TableValueReader tvr))
{
var ptr = tvr.Read(2, out int size);
var doc = new BlittableJsonReaderObject(ptr, size, context);
var existingSubscriptionState = JsonDeserializationClient.SubscriptionState(doc);
if (SubscriptionId != existingSubscriptionState.SubscriptionId)
{
throw new RachisApplyException("A subscription could not be modified because the name '" + subscriptionItemName +
"' is already in use in a subscription with different Id.");
}
if (string.IsNullOrEmpty(InitialChangeVector) == false && InitialChangeVector == nameof(Constants.Documents.SubscriptionChangeVectorSpecialStates.DoNotChange))
{
InitialChangeVector = existingSubscriptionState.ChangeVectorForNextBatchStartingPoint;
}
else
{
AssertValidChangeVector();
}
}
else
{
AssertValidChangeVector();
}
using (var receivedSubscriptionState = context.ReadObject(new SubscriptionState
{
Query = Query,
ChangeVectorForNextBatchStartingPoint = InitialChangeVector,
SubscriptionId = subscriptionId,
SubscriptionName = SubscriptionName,
LastBatchAckTime = null,
Disabled = Disabled,
MentorNode = MentorNode,
LastClientConnectionTime = null
}.ToJson(), SubscriptionName))
{
ClusterStateMachine.UpdateValue(subscriptionId, items, valueNameLowered, valueName, receivedSubscriptionState);
}
}
}
public void FullBackupTo(string backupPath)
{
using (var file = new FileStream(backupPath, FileMode.Create))
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var databaseRecord = _serverStore.Cluster.ReadDatabase(context, Name);
Debug.Assert(databaseRecord != null);
var zipArchiveEntry = package.CreateEntry(RestoreSettings.FileName, CompressionLevel.Optimal);
using (var zipStream = zipArchiveEntry.Open())
using (var writer = new BlittableJsonTextWriter(context, zipStream))
{
//TODO: encrypt this file using the MasterKey
//http://issues.hibernatingrhinos.com/issue/RavenDB-7546
writer.WriteStartObject();
// save the database record
writer.WritePropertyName(nameof(RestoreSettings.DatabaseRecord));
var databaseRecordBlittable = EntityToBlittable.ConvertEntityToBlittable(databaseRecord, DocumentConventions.Default, context);
context.Write(writer, databaseRecordBlittable);
// save the database values (subscriptions, periodic backups statuses, etl states...)
writer.WriteComma();
writer.WritePropertyName(nameof(RestoreSettings.DatabaseValues));
writer.WriteStartObject();
var first = true;
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context,
Helpers.ClusterStateMachineValuesPrefix(Name)))
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(keyValue.Key.ToString());
context.Write(writer, keyValue.Value);
}
writer.WriteEndObject();
// end of dictionary
writer.WriteEndObject();
}
BackupMethods.Full.ToFile(GetAllStoragesEnvironmentInformation(), package);
file.Flush(true); // make sure that we fully flushed to disk
}
}
public IEnumerable <SubscriptionGeneralDataAndStats> GetAllSubscriptions(TransactionOperationContext serverStoreContext, bool history, int start, int take)
{
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(serverStoreContext,
SubscriptionState.SubscriptionPrefix(_db.Name)))
{
var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value);
var subscriptionGeneralData = new SubscriptionGeneralDataAndStats(subscriptionState);
GetSubscriptionInternal(subscriptionGeneralData, history);
yield return(subscriptionGeneralData);
}
}
public void Test_Primary_BecomeStandAlone_WhenInitialConnectToBackup_Fails()
{
StateChanged <InstanceState> stateChanged = null;
var observer = new MockStateObserver(s => stateChanged = s, Assert.IsNull);
InstanceState localState;
ClusterStateMachine machine = CreateClusterStateMachine(observer, NodeRole.Primary, out localState, becomeActiveWhenPrimaryOnInitialConnectionTimeout: true);
machine.RaiseEvent(localState, machine.LostPartner, localState);
Assert.AreEqual("Connecting", stateChanged.Previous.Name);
Assert.AreEqual("Active", stateChanged.Current.Name);
}
private static ClusterStateMachine CreateClusterStateMachine(MockStateObserver observer, NodeRole role,
out InstanceState localState, Action <ClusterException> clusterExceptionAction = null, bool becomeActiveWhenPrimaryOnInitialConnectionTimeout = false)
{
var machine = new ClusterStateMachine(clusterExceptionAction ?? Assert.IsNull, s => true, becomeActiveWhenPrimaryOnInitialConnectionTimeout);
machine.StateChanged.Subscribe(observer);
localState = new InstanceState {
Role = role
};
machine.RaiseEvent(localState, machine.Start);
return(machine);
}
public void HeartbeatDoesntChangeRecomputeTime()
{
var clusterState = new ClusterStateMachine();
MachineId n1Id;
(clusterState, n1Id) = clusterState.RegisterMachine(new MachineLocation("node1"), _clock.UtcNow);
_clock.Increment(TimeSpan.FromMinutes(1));
clusterState = clusterState.Heartbeat(n1Id, _clock.UtcNow, MachineState.Open).ThrowIfFailure().Next;
clusterState.LastStateMachineRecomputeTimeUtc.Should().Be(DateTime.MinValue);
}
public void RegisterMachineEmitsIdsInSequence()
{
var clusterState = new ClusterStateMachine();
var nowUtc = _clock.UtcNow;
MachineId machineId;
(clusterState, machineId) = clusterState.RegisterMachine(new MachineLocation("node1"), nowUtc);
machineId.Index.Should().Be(1);
(_, machineId) = clusterState.RegisterMachine(new MachineLocation("node2"), nowUtc);
machineId.Index.Should().Be(2);
}
public static ClusterManager Create(DocumentDatabase systemDatabase, DatabasesLandlord databasesLandlord)
{
if (systemDatabase == null)
{
throw new ArgumentNullException("systemDatabase");
}
if (databasesLandlord == null)
{
throw new ArgumentNullException("databasesLandlord");
}
DatabaseHelper.AssertSystemDatabase(systemDatabase);
var configuration = systemDatabase.Configuration;
var nodeConnectionInfo = CreateSelfConnection(systemDatabase);
StorageEnvironmentOptions options;
if (configuration.Core.RunInMemory == false)
{
var directoryPath = Path.Combine(configuration.Core.DataDirectory ?? AppDomain.CurrentDomain.BaseDirectory, "Raft");
if (Directory.Exists(directoryPath) == false)
{
Directory.CreateDirectory(directoryPath);
}
options = StorageEnvironmentOptions.ForPath(directoryPath);
}
else
{
options = StorageEnvironmentOptions.CreateMemoryOnly(configuration.Storage.TempPath);
}
var transport = new HttpTransport(nodeConnectionInfo.Name, systemDatabase.WorkContext.CancellationToken);
var stateMachine = new ClusterStateMachine(systemDatabase, databasesLandlord);
var raftEngineOptions = new RaftEngineOptions(nodeConnectionInfo, options, transport, stateMachine)
{
ElectionTimeout = (int)configuration.Cluster.ElectionTimeout.AsTimeSpan.TotalMilliseconds,
HeartbeatTimeout = (int)configuration.Cluster.HeartbeatTimeout.AsTimeSpan.TotalMilliseconds,
MaxLogLengthBeforeCompaction = configuration.Cluster.MaxLogLengthBeforeCompaction,
MaxEntriesPerRequest = configuration.Cluster.MaxEntriesPerRequest,
MaxStepDownDrainTime = configuration.Cluster.MaxStepDownDrainTime.AsTimeSpan
};
var raftEngine = new RaftEngine(raftEngineOptions);
stateMachine.RaftEngine = raftEngine;
return(new ClusterManager(raftEngine));
}
public void HeartbeatUpdatesLastHeartbeatTimeAndState()
{
var clusterState = new ClusterStateMachine();
MachineId machineId;
(clusterState, machineId) = clusterState.RegisterMachine(new MachineLocation("node1"), _clock.UtcNow);
_clock.Increment(TimeSpan.FromMinutes(1));
clusterState = clusterState.Heartbeat(machineId, _clock.UtcNow, MachineState.Open).ThrowIfFailure().Next;
var r = clusterState.GetStatus(machineId).ShouldBeSuccess().Value;
r.LastHeartbeatTimeUtc.Should().Be(_clock.UtcNow);
r.State.Should().Be(MachineState.Open);
}
public static List <string> GetDatabases(UpdateStep step)
{
var dbs = new List <string>();
using (var items = step.ReadTx.OpenTable(ClusterStateMachine.ItemsSchema, ClusterStateMachine.Items))
using (Slice.From(step.ReadTx.Allocator, DbKey, out var loweredPrefix))
{
foreach (var result in items.SeekByPrimaryKeyPrefix(loweredPrefix, Slices.Empty, 0))
{
dbs.Add(ClusterStateMachine.GetCurrentItemKey(result.Value).Substring(DbKey.Length));
}
}
return(dbs);
}
public void Test_Backup_Found_Primary()
{
StateChanged <InstanceState> stateChanged = null;
var observer = new MockStateObserver(s => stateChanged = s, Assert.IsNull);
InstanceState localState;
ClusterStateMachine machine = CreateClusterStateMachine(observer, NodeRole.Backup, out localState);
var remoteState = new NodeState {
Role = NodeRole.Primary, Status = NodeStatus.Connecting
};
machine.RaiseEvent(localState, machine.PartnerStatusReceived, remoteState);
Assert.AreEqual("Connecting", stateChanged.Previous.Name);
Assert.AreEqual("Passive", stateChanged.Current.Name);
}
private static long GetSubscriptionCountOnNode(DocumentDatabase database, DatabaseRecord dbRecord, ServerStore serverStore, TransactionOperationContext context)
{
long taskCountOnNode = 0;
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(database.Name)))
{
var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value);
var taskTag = database.WhoseTaskIsIt(dbRecord.Topology, subscriptionState, subscriptionState);
if (serverStore.NodeTag == taskTag)
{
taskCountOnNode++;
}
}
return(taskCountOnNode);
}
public void RecomputeDoesntRunIfNotNeeded()
{
var now = _clock.UtcNow;
var current = new ClusterStateMachine()
{
LastStateMachineRecomputeTimeUtc = now,
};
var cfg = new ClusterStateRecomputeConfiguration()
{
RecomputeFrequency = TimeSpan.FromMilliseconds(1),
};
var next = current.Recompute(cfg, now);
next.Should().BeEquivalentTo(current);
}
internal static bool UpdateCertificatesTableInternal(UpdateStep step)
{
var schema = ClusterStateMachine.CertificatesSchema;
var readCertsTable = step.ReadTx.OpenTable(schema, ClusterStateMachine.CertificatesSlice);
var writeCertsTable = step.WriteTx.OpenTable(schema, ClusterStateMachine.CertificatesSlice);
using (var context = JsonOperationContext.ShortTermSingleUse())
{
foreach (var cert in readCertsTable.SeekByPrimaryKey(Slices.Empty, 0))
{
(string key, BlittableJsonReaderObject doc) = GetCurrentItem(step.WriteTx, context, cert);
using (doc)
{
var def = JsonDeserializationServer.CertificateDefinition(doc);
From11.DropCertificatePrefixFromDefinition(def, out var touched);
var loweredKey = key.ToLowerInvariant();
if (loweredKey != key)
{
// we have upper cased key (thumbprint)
// let's remove current record from table and force writing it again with lower cased key value
using (Slice.From(step.WriteTx.Allocator, key, out Slice keySlice))
writeCertsTable.DeleteByKey(keySlice);
touched = true;
}
if (touched)
{
using (Slice.From(step.WriteTx.Allocator, def.PublicKeyPinningHash, out Slice hashSlice))
using (Slice.From(step.WriteTx.Allocator, loweredKey, out Slice keySlice))
using (var newCert = context.ReadObject(def.ToJson(), "certificate/updated"))
{
ClusterStateMachine.UpdateCertificate(writeCertsTable, keySlice, hashSlice, newCert);
}
}
}
}
}
return(true);
}
public string GetSubscriptionNameById(TransactionOperationContext serverStoreContext, long id)
{
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(serverStoreContext,
SubscriptionState.SubscriptionPrefix(_db.Name)))
{
if (keyValue.Value.TryGet(nameof(SubscriptionState.SubscriptionId), out long _id) == false)
{
continue;
}
if (_id == id)
{
if (keyValue.Value.TryGet(nameof(SubscriptionState.SubscriptionName), out string name))
{
return(name);
}
}
}
return(null);
}
public void Test_LostPartner_During_Any_Should_LogException()
{
StateChanged <InstanceState> stateChanged = null;
var observer = new MockStateObserver(s => stateChanged = s, Assert.IsNull);
InstanceState localState;
ClusterException cex = null;
ClusterStateMachine machine = CreateClusterStateMachine(observer, NodeRole.Primary, out localState,
ex => cex = ex);
var remoteState = new NodeState {
Role = NodeRole.Backup, Status = NodeStatus.Passive
};
machine.RaiseEvent(localState, machine.PartnerStatusReceived, remoteState);
machine.RaiseEvent(localState, machine.LostPartner, localState);
Assert.AreEqual(ClusterFailureReason.LostPartner, cex.Reason);
Assert.AreEqual("Connecting", stateChanged.Previous.Name);
Assert.AreEqual("Active", stateChanged.Current.Name);
}
public void HeartbeatKeepsOtherRecordsAsIs()
{
var clusterState = new ClusterStateMachine();
var nowUtc = _clock.UtcNow;
MachineId n1Id;
(clusterState, n1Id) = clusterState.RegisterMachine(new MachineLocation("node1"), nowUtc);
MachineId n2Id;
(clusterState, n2Id) = clusterState.RegisterMachine(new MachineLocation("node2"), nowUtc);
_clock.Increment(TimeSpan.FromMinutes(1));
clusterState = clusterState.Heartbeat(n1Id, _clock.UtcNow, MachineState.Closed).ThrowIfFailure().Next;
var r = clusterState.GetStatus(n2Id).ShouldBeSuccess().Value;
r.LastHeartbeatTimeUtc.Should().Be(nowUtc);
r.State.Should().Be(MachineState.Open);
}
public void RegisterNewMachineUsesCorrectDefaults()
{
var clusterState = new ClusterStateMachine();
var nowUtc = _clock.UtcNow;
MachineId machineId;
var machineLocation = new MachineLocation("node1");
(clusterState, machineId) = clusterState.RegisterMachine(machineLocation, nowUtc);
machineId.Index.Should().Be(MachineId.MinValue);
var record = clusterState.GetStatus(machineId).ThrowIfFailure();
record.Should().BeEquivalentTo(new MachineRecord()
{
Id = new MachineId(MachineId.MinValue),
Location = machineLocation,
State = ClusterStateMachine.InitialState,
LastHeartbeatTimeUtc = nowUtc,
});
}
private IEnumerable <OngoingTask> CollectSubscriptionTasks(TransactionOperationContext context, DatabaseRecord databaseRecord, ClusterTopology clusterTopology)
{
foreach (var keyValue in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(databaseRecord.DatabaseName)))
{
var subscriptionState = JsonDeserializationClient.SubscriptionState(keyValue.Value);
var tag = databaseRecord.Topology.WhoseTaskIsIt(subscriptionState, ServerStore.Engine.CurrentState);
yield return(new OngoingTaskSubscription
{
// Supply only needed fields for List View
ResponsibleNode = new NodeId
{
NodeTag = tag,
NodeUrl = clusterTopology.GetUrlFromTag(tag)
},
TaskName = subscriptionState.SubscriptionName,
TaskState = subscriptionState.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
TaskId = subscriptionState.SubscriptionId,
Query = subscriptionState.Query
});
}
}
private static ClusterStateMachine CreateClusterStateMachine(MockStateObserver observer, NodeRole role,
out InstanceState localState, Action<ClusterException> clusterExceptionAction = null)
{
var machine = new ClusterStateMachine(clusterExceptionAction ?? Assert.IsNull, s => true);
machine.StateChanged.Subscribe(observer);
localState = new InstanceState {Role = role};
machine.RaiseEvent(localState, machine.Start);
return machine;
}
private static ClusterStateMachine CreateClusterStateMachine(MockStateObserver observer, NodeRole role,
out InstanceState localState, Action<ClusterException> clusterExceptionAction = null, bool becomeActiveWhenPrimaryOnInitialConnectionTimeout = false)
{
var machine = new ClusterStateMachine(clusterExceptionAction ?? Assert.IsNull, s => true, becomeActiveWhenPrimaryOnInitialConnectionTimeout);
machine.StateChanged.Subscribe(observer);
localState = new InstanceState { Role = role };
machine.RaiseEvent(localState, machine.Start);
return machine;
}