public void Follower_as_a_single_node_becomes_leader_automatically()
		{
			var hub = new InMemoryTransportHub();
			var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
			storageEnvironmentOptions.OwnsPagers = false;
			
			var raftEngineOptions = new RaftEngineOptions(
				new NodeConnectionInfo { Name = "node1" }, 
				storageEnvironmentOptions,
				hub.CreateTransportFor("node1"),
				new DictionaryStateMachine()
				)
			{
				ElectionTimeout = 1000,
				HeartbeatTimeout = 1000/6
			};

			PersistentState.ClusterBootstrap(raftEngineOptions);
			storageEnvironmentOptions.OwnsPagers = true;

			using (var raftNode = new RaftEngine(raftEngineOptions))
			{
				Assert.Equal(RaftEngineState.Leader, raftNode.State);
			}
		}
Beispiel #2
0
 public override async Task <HttpResponseMessage> ExecuteAsync(HttpControllerContext controllerContext, CancellationToken cancellationToken)
 {
     RaftEngine   = (RaftEngine)controllerContext.Configuration.Properties[typeof(RaftEngine)];
     StateMachine = (KeyValueStateMachine)RaftEngine.StateMachine;
     try
     {
         return(await base.ExecuteAsync(controllerContext, cancellationToken));
     }
     catch (NotLeadingException)
     {
         var currentLeader = RaftEngine.CurrentLeader;
         if (currentLeader == null)
         {
             return(Request.CreateErrorResponse(HttpStatusCode.PreconditionFailed, "No current leader, try again later"));
         }
         var leaderNode = RaftEngine.CurrentTopology.GetNodeByName(currentLeader);
         if (leaderNode == null)
         {
             return(Request.CreateErrorResponse(HttpStatusCode.PreconditionFailed, "Current leader " + currentLeader + " is not found in the topology. This should not happen."));
         }
         return(new HttpResponseMessage(HttpStatusCode.Redirect)
         {
             Headers =
             {
                 Location = leaderNode.Uri
             }
         });
     }
 }
Beispiel #3
0
        public void Follower_as_a_single_node_becomes_leader_automatically()
        {
            var hub = new InMemoryTransportHub();
            var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();

            storageEnvironmentOptions.OwnsPagers = false;

            var raftEngineOptions = new RaftEngineOptions(
                new NodeConnectionInfo {
                Name = "node1"
            },
                storageEnvironmentOptions,
                hub.CreateTransportFor("node1"),
                new DictionaryStateMachine()
                )
            {
                ElectionTimeout  = 1000,
                HeartbeatTimeout = 1000 / 6
            };

            PersistentState.ClusterBootstrap(raftEngineOptions);
            storageEnvironmentOptions.OwnsPagers = true;

            using (var raftNode = new RaftEngine(raftEngineOptions))
            {
                Assert.Equal(RaftEngineState.Leader, raftNode.State);
            }
        }
Beispiel #4
0
        protected RaftEngine NewNodeFor(RaftEngine leader)
        {
            var raftEngine = new RaftEngine(CreateNodeOptions("node" + _nodes.Count, leader.Options.ElectionTimeout, StorageEnvironmentOptions.CreateMemoryOnly()));

            _nodes.Add(raftEngine);
            return(raftEngine);
        }
Beispiel #5
0
        protected ManualResetEventSlim WaitForSnapshot(RaftEngine node)
        {
            var cde = new ManualResetEventSlim();

            node.CreatedSnapshot += cde.Set;
            return(cde);
        }
Beispiel #6
0
        protected ManualResetEventSlim WaitForSnapshotInstallation(RaftEngine node)
        {
            var cde = new ManualResetEventSlim();

            node.SnapshotInstalled += cde.Set;
            return(cde);
        }
Beispiel #7
0
        protected AbstractRaftStateBehavior(RaftEngine engine)
        {
            Engine            = engine;
            LastHeartbeatTime = DateTime.UtcNow;
            _log = LogManager.GetLogger(engine.Name + "." + GetType().Name);

            _actionDispatch = new Dictionary <Type, Action <MessageContext> >
            {
                { typeof(RequestVoteRequest), ctx => ctx.Reply(Handle((RequestVoteRequest)ctx.Message)) },
                { typeof(AppendEntriesRequest), ctx => ctx.Reply(Handle((AppendEntriesRequest)ctx.Message)) },
                { typeof(CanInstallSnapshotRequest), ctx => ctx.Reply(Handle((CanInstallSnapshotRequest)ctx.Message)) },
                { typeof(InstallSnapshotRequest), ctx =>
                  {
                      var reply = Handle(ctx, (InstallSnapshotRequest)ctx.Message, ctx.Stream);
                      if (reply != null)
                      {
                          ctx.Reply(reply);
                      }
                      else
                      {
                          ctx.AsyncResponse = true;
                      }
                  } },

                { typeof(RequestVoteResponse), ctx => Handle((RequestVoteResponse)ctx.Message) },
                { typeof(AppendEntriesResponse), ctx => Handle((AppendEntriesResponse)ctx.Message) },
                { typeof(CanInstallSnapshotResponse), ctx => Handle((CanInstallSnapshotResponse)ctx.Message) },
                { typeof(InstallSnapshotResponse), ctx => Handle((InstallSnapshotResponse)ctx.Message) },

                { typeof(NothingToDo), ctx => { } },
                { typeof(TimeoutNowRequest), ctx => Handle((TimeoutNowRequest)ctx.Message) },
                { typeof(DisconnectedFromCluster), ctx => Handle((DisconnectedFromCluster)ctx.Message) },
                { typeof(Action), ctx => ((Action)ctx.Message)() },
            };
        }
Beispiel #8
0
        public HttpTransportPingTest()
        {
            _node1Transport = new HttpTransport("node1", _requestsTimeout, CancellationToken.None);

            var node1 = new NodeConnectionInfo {
                Name = "node1", Uri = new Uri("http://localhost:9079")
            };
            var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine())
            {
                ElectionTimeout  = 60 * 1000,
                HeartbeatTimeout = 10 * 1000
            };

            _raftEngine = new RaftEngine(engineOptions);

            NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(9079);

            _server = WebApp.Start(new StartOptions
            {
                Urls = { "http://+:9079/" }
            }, builder =>
            {
                var httpConfiguration = new HttpConfiguration();
                RaftWebApiConfig.Load();
                httpConfiguration.MapHttpAttributeRoutes();
                httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus;
                builder.UseWebApi(httpConfiguration);
            });
        }
Beispiel #9
0
        public RaftNode(int port)
        {
            _name = "node-" + port;
            _url  = string.Format("http://{0}:{1}", Environment.MachineName, port);

            var nodeTransport = new HttpTransport(_name);

            var node1 = new NodeConnectionInfo {
                Name = _name, Uri = new Uri(_url)
            };
            var engineOptions = new RaftEngineOptions(
                node1,
                StorageEnvironmentOptions.CreateMemoryOnly(),
                nodeTransport,
                new DictionaryStateMachine());

            engineOptions.ElectionTimeout  *= 2;
            engineOptions.HeartbeatTimeout *= 2;

            _raftEngine = new RaftEngine(engineOptions);

            _server = WebApp.Start(new StartOptions
            {
                Urls = { string.Format("http://+:{0}/", port) }
            }, builder =>
            {
                var httpConfiguration = new HttpConfiguration();
                RaftWebApiConfig.Load();
                httpConfiguration.MapHttpAttributeRoutes();
                httpConfiguration.Properties[typeof(HttpTransportBus)] = nodeTransport.Bus;
                builder.UseWebApi(httpConfiguration);
            });
        }
Beispiel #10
0
        protected AbstractRaftStateBehavior(RaftEngine engine)
        {
            Engine = engine;
            LastHeartbeatTime = DateTime.UtcNow;
            _log = LogManager.GetLogger(engine.Name + "." + GetType().Name);

            _actionDispatch = new Dictionary<Type, Action<MessageContext>>
            {
                {typeof (RequestVoteRequest), ctx => ctx.Reply(Handle((RequestVoteRequest) ctx.Message))},
                {typeof (AppendEntriesRequest), ctx => ctx.Reply(Handle((AppendEntriesRequest) ctx.Message))},
                {typeof (CanInstallSnapshotRequest), ctx => ctx.Reply(Handle((CanInstallSnapshotRequest) ctx.Message))},
                {typeof (InstallSnapshotRequest), ctx =>
                {
                    var reply = Handle(ctx, (InstallSnapshotRequest) ctx.Message, ctx.Stream);
                    if (reply != null)
                        ctx.Reply(reply);
                    else
                        ctx.AsyncResponse = true;
                }},

                {typeof(RequestVoteResponse), ctx => Handle((RequestVoteResponse)ctx.Message)},
                {typeof(AppendEntriesResponse), ctx => Handle((AppendEntriesResponse)ctx.Message)},
                {typeof(CanInstallSnapshotResponse), ctx => Handle((CanInstallSnapshotResponse)ctx.Message)},
                {typeof(InstallSnapshotResponse), ctx => Handle((InstallSnapshotResponse)ctx.Message)},

                {typeof(NothingToDo), ctx => { }},
                {typeof (TimeoutNowRequest), ctx => Handle((TimeoutNowRequest) ctx.Message)},
                {typeof (DisconnectedFromCluster), ctx => Handle((DisconnectedFromCluster) ctx.Message)},
                {typeof (Action), ctx => ((Action)ctx.Message)()},

            };
        }
	    public FollowerStateBehavior(RaftEngine engine, bool avoidLeadership) : base(engine)
	    {
		    _avoidLeadership = avoidLeadership;
		    _currentTermWhenWeBecameFollowers = engine.PersistentState.CurrentTerm + 1;// we are going to have a new term immediately.
		    var random = new Random(Engine.Name.GetHashCode() ^ (int)DateTime.Now.Ticks);
			Timeout = random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
	    }
Beispiel #12
0
        public FollowerStateBehavior(RaftEngine engine, bool avoidLeadership) : base(engine)
        {
            _avoidLeadership = avoidLeadership;
            _currentTermWhenWeBecameFollowers = engine.PersistentState.CurrentTerm + 1;        // we are going to have a new term immediately.
            var random = new Random(Engine.Name.GetHashCode() ^ (int)DateTime.Now.Ticks);

            Timeout = random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
        }
 public CandidateStateBehavior(RaftEngine engine, bool forcedElection)
     : base(engine)
 {
     _forcedElection   = forcedElection;
     _wonTrialElection = forcedElection;
     _random           = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
     Timeout           = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
 }
Beispiel #14
0
        public async Task <HttpResponseMessage> Leave([FromUri] string name)
        {
            await RaftEngine.RemoveFromClusterAsync(new NodeConnectionInfo
            {
                Name = name
            });

            return(new HttpResponseMessage(HttpStatusCode.Accepted));
        }
		public CandidateStateBehavior(RaftEngine engine, bool forcedElection)
			: base(engine)
		{
			_forcedElection = forcedElection;
			_wonTrialElection = forcedElection;
			_random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
			Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
			StartElection();
		}
Beispiel #16
0
		protected ManualResetEventSlim WaitForToplogyChange(RaftEngine node)
		{
			var mre = new ManualResetEventSlim();
			node.TopologyChanged += state =>
			{
				if (node.CurrentTopology.HasVoters)
					mre.Set();
			};
			return mre;
		}
Beispiel #17
0
		protected ManualResetEventSlim WaitForStateChange(RaftEngine node, RaftEngineState requestedState)
		{
			var mre = new ManualResetEventSlim();
			node.StateChanged += state =>
			{
				if (state == requestedState)
					mre.Set();
			};
			return mre;
		}
Beispiel #18
0
 public ClusterManager(RaftEngine engine, DatabasesLandlord databasesLandlord)
 {
     Engine                     = engine;
     Client                     = new ClusterManagementHttpClient(engine);
     DatabasesLandlord          = databasesLandlord;
     engine.StateChanged       += OnRaftEngineStateChanged;
     engine.ProposingCandidacy += OnProposingCandidacy;
     maxReplicationLatency      = DatabasesLandlord.SystemDatabase.Configuration.Cluster.MaxReplicationLatency;
     clusterManagerStartTime    = DateTime.UtcNow;
 }
Beispiel #19
0
 protected void RestartAllNodes()
 {
     foreach (var raftEngine in _nodes)
     {
         raftEngine.Options.StorageOptions.OwnsPagers = false;
         raftEngine.Dispose();
     }
     for (int i = 0; i < _nodes.Count; i++)
     {
         _nodes[i] = new RaftEngine(_nodes[i].Options);
     }
 }
Beispiel #20
0
        public async Task <HttpResponseMessage> Read([FromUri] string key, [FromUri] string mode = null)
        {
            switch (mode)
            {
            case "quorum":
                var taskCompletionSource = new TaskCompletionSource <object>();
                try
                {
                    RaftEngine.AppendCommand(new GetCommand
                    {
                        Key        = key,
                        Completion = taskCompletionSource
                    });
                }
                catch (NotLeadingException e)
                {
                    return(RedirectToLeader(e.CurrentLeader, Request.RequestUri));
                }
                var consistentRead = await taskCompletionSource.Task;
                return(Request.CreateResponse(HttpStatusCode.OK, new
                {
                    RaftEngine.State,
                    Key = key,
                    Value = consistentRead,
                    Missing = consistentRead == null
                }));

            case "leader":
                if (RaftEngine.State != RaftEngineState.Leader)
                {
                    return(RedirectToLeader(RaftEngine.CurrentLeader, Request.RequestUri));
                }
                goto case null;

            case "any":
            case null:
                var read = StateMachine.Read(key);
                return(Request.CreateResponse(HttpStatusCode.OK, new
                {
                    RaftEngine.State,
                    Key = key,
                    Value = read,
                    Missing = read == null
                }));

            default:
                return(Request.CreateResponse(HttpStatusCode.BadRequest, new
                {
                    Error = "Unknown read mode"
                }));
            }
        }
Beispiel #21
0
        protected ManualResetEventSlim WaitForNodeToBecomeVoter(RaftEngine node)
        {
            var mre = new ManualResetEventSlim();

            node.TopologyChanged += state =>
            {
                if (node.CurrentTopology.AllVotingNodes.Select(ni => ni.Name).Contains(node.Name))
                {
                    mre.Set();
                }
            };
            return(mre);
        }
Beispiel #22
0
        protected ManualResetEventSlim WaitForStateChange(RaftEngine node, RaftEngineState requestedState)
        {
            var mre = new ManualResetEventSlim();

            node.StateChanged += state =>
            {
                if (state == requestedState)
                {
                    mre.Set();
                }
            };
            return(mre);
        }
Beispiel #23
0
        protected ManualResetEventSlim WaitForToplogyChange(RaftEngine node)
        {
            var mre = new ManualResetEventSlim();

            node.TopologyChanged += state =>
            {
                if (node.CurrentTopology.HasVoters)
                {
                    mre.Set();
                }
            };
            return(mre);
        }
        public static ClusterManager Create(DocumentDatabase systemDatabase, DatabasesLandlord databasesLandlord)
        {
            if (systemDatabase == null)
            {
                throw new ArgumentNullException("systemDatabase");
            }

            if (databasesLandlord == null)
            {
                throw new ArgumentNullException("databasesLandlord");
            }

            DatabaseHelper.AssertSystemDatabase(systemDatabase);

            var configuration      = systemDatabase.Configuration;
            var nodeConnectionInfo = CreateSelfConnection(systemDatabase);

            StorageEnvironmentOptions options;

            if (configuration.Core.RunInMemory == false)
            {
                var directoryPath = Path.Combine(configuration.Core.DataDirectory ?? AppDomain.CurrentDomain.BaseDirectory, "Raft");
                if (Directory.Exists(directoryPath) == false)
                {
                    Directory.CreateDirectory(directoryPath);
                }

                options = StorageEnvironmentOptions.ForPath(directoryPath);
            }
            else
            {
                options = StorageEnvironmentOptions.CreateMemoryOnly(configuration.Storage.TempPath);
            }

            var transport         = new HttpTransport(nodeConnectionInfo.Name, systemDatabase.WorkContext.CancellationToken);
            var stateMachine      = new ClusterStateMachine(systemDatabase, databasesLandlord);
            var raftEngineOptions = new RaftEngineOptions(nodeConnectionInfo, options, transport, stateMachine)
            {
                ElectionTimeout              = (int)configuration.Cluster.ElectionTimeout.AsTimeSpan.TotalMilliseconds,
                HeartbeatTimeout             = (int)configuration.Cluster.HeartbeatTimeout.AsTimeSpan.TotalMilliseconds,
                MaxLogLengthBeforeCompaction = configuration.Cluster.MaxLogLengthBeforeCompaction,
                MaxEntriesPerRequest         = configuration.Cluster.MaxEntriesPerRequest,
                MaxStepDownDrainTime         = configuration.Cluster.MaxStepDownDrainTime.AsTimeSpan
            };
            var raftEngine = new RaftEngine(raftEngineOptions);

            stateMachine.RaftEngine = raftEngine;

            return(new ClusterManager(raftEngine));
        }
Beispiel #25
0
        public async Task <HttpResponseMessage> Join([FromUri] string url, [FromUri] string name)
        {
            var uri = new Uri(url);

            name = name ?? uri.Host + (uri.IsDefaultPort ? "" : ":" + uri.Port);

            await RaftEngine.AddToClusterAsync(new NodeConnectionInfo
            {
                Name = name,
                Uri  = uri
            });

            return(new HttpResponseMessage(HttpStatusCode.Accepted));
        }
		public SteppingDownStateBehavior(RaftEngine engine)
			: base(engine)
		{
			_stepdownDuration = Stopwatch.StartNew();
			// we are sending this to ourselves because we want to make 
			// sure that we immediately check if we can step down
			Engine.Transport.SendToSelf(new AppendEntriesResponse
			{
				CurrentTerm = Engine.PersistentState.CurrentTerm,
				From = Engine.Name,
				ClusterTopologyId = Engine.CurrentTopology.TopologyId,
				LastLogIndex = Engine.PersistentState.LastLogEntry().Index,
				LeaderId = Engine.Name,
				Message = "Forcing step down evaluation",
				Success = true
			});
		}
Beispiel #27
0
 public SteppingDownStateBehavior(RaftEngine engine)
     : base(engine)
 {
     _stepdownDuration = Stopwatch.StartNew();
     // we are sending this to ourselves because we want to make
     // sure that we immediately check if we can step down
     Engine.Transport.SendToSelf(new AppendEntriesResponse
     {
         CurrentTerm       = Engine.PersistentState.CurrentTerm,
         From              = Engine.Name,
         ClusterTopologyId = Engine.CurrentTopology.TopologyId,
         LastLogIndex      = Engine.PersistentState.LastLogEntry().Index,
         LeaderId          = Engine.Name,
         Message           = "Forcing step down evaluation",
         Success           = true
     });
 }
Beispiel #28
0
        public LeaderStateBehavior(RaftEngine engine)
            : base(engine)
        {
            Timeout = engine.Options.ElectionTimeout / 2;
            engine.TopologyChanged += OnTopologyChanged;
            var lastLogEntry = Engine.PersistentState.LastLogEntry();

            foreach (var peer in Engine.CurrentTopology.AllNodeNames)
            {
                _nextIndexes[peer] = lastLogEntry.Index + 1;
                _matchIndexes[peer] = 0;
            }

            AppendCommand(new NopCommand());
            _stopHeartbeatCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(Engine.CancellationToken, _disposedCancellationTokenSource.Token);

            _heartbeatTask = Task.Run(() => Heartbeat(), _stopHeartbeatCancellationTokenSource.Token);
        }
Beispiel #29
0
        public LeaderStateBehavior(RaftEngine engine)
            : base(engine)
        {
            Timeout = engine.Options.ElectionTimeout / 2;
            engine.TopologyChanged += OnTopologyChanged;
            var lastLogEntry = Engine.PersistentState.LastLogEntry();

            foreach (var peer in Engine.CurrentTopology.AllNodeNames)
            {
                _nextIndexes[peer]  = lastLogEntry.Index + 1;
                _matchIndexes[peer] = 0;
            }

            AppendCommand(new NopCommand());
            _stopHeartbeatCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(Engine.CancellationToken, _disposedCancellationTokenSource.Token);

            _heartbeatTask = Task.Factory.StartNew(Heartbeat, _stopHeartbeatCancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
        }
Beispiel #30
0
		protected ManualResetEventSlim WaitForCommit(RaftEngine node, Func<DictionaryStateMachine, bool> predicate)
		{
			var cde = new ManualResetEventSlim();
			node.CommitApplied += command =>
			{
				if (predicate((DictionaryStateMachine)node.StateMachine))
					cde.Set();
			};
			node.SnapshotInstalled += () =>
			{
				var state = (DictionaryStateMachine)node.StateMachine;
				if (predicate(state))
				{
					cde.Set();
				}
			};
			return cde;
		}
Beispiel #31
0
        private async Task <HttpResponseMessage> Batch(KeyValueOperation[] operations)
        {
            var taskCompletionSource = new TaskCompletionSource <object>();

            try
            {
                RaftEngine.AppendCommand(new OperationBatchCommand
                {
                    Batch      = operations,
                    Completion = taskCompletionSource
                });
            }
            catch (NotLeadingException e)
            {
                return(RedirectToLeader(e.CurrentLeader, Request.RequestUri));
            }
            await taskCompletionSource.Task;

            return(Request.CreateResponse(HttpStatusCode.Accepted));
        }
Beispiel #32
0
        public void AppendToLog(RaftEngine engine, IEnumerable <LogEntry> entries, long removeAllAfter)
        {
            using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
            {
                var logs     = tx.ReadTree(LogsTreeName);
                var terms    = tx.ReadTree(EntryTermsTreeName);
                var metadata = tx.ReadTree(MetadataTreeName);
                var changed  = ReadIsTopologyChanged(metadata);
                if (changed > removeAllAfter)
                {
                    // need to reset the topology
                    var current         = metadata.Read("current-topology");
                    var prevTopologyStr = current == null ? "{}" : current.Reader.ToStringValue();
                    metadata.Add("current-topology", prevTopologyStr);
                    metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(0L));
                    var prevTopology = JsonConvert.DeserializeObject <Topology>(prevTopologyStr);
                    engine.RevertTopologyTo(prevTopology);
                }

                using (var it = logs.Iterate())
                {
                    if (it.Seek(new Slice(EndianBitConverter.Big.GetBytes(removeAllAfter))) &&
                        it.MoveNext())
                    {
                        do
                        {
                            terms.Delete(it.CurrentKey);
                        } while (it.DeleteCurrentAndMoveNext());
                    }
                }

                foreach (var entry in entries)
                {
                    var key = new Slice(EndianBitConverter.Big.GetBytes(entry.Index));
                    logs.Add(key, entry.Data);
                    terms.Add(key, new Slice(BitConverter.GetBytes(entry.Term)));
                }

                tx.Commit();
            }
        }
Beispiel #33
0
        protected ManualResetEventSlim WaitForCommit(RaftEngine node, Func <DictionaryStateMachine, bool> predicate)
        {
            var cde = new ManualResetEventSlim();

            node.CommitApplied += command =>
            {
                if (predicate((DictionaryStateMachine)node.StateMachine))
                {
                    cde.Set();
                }
            };
            node.SnapshotInstalled += () =>
            {
                var state = (DictionaryStateMachine)node.StateMachine;
                if (predicate(state))
                {
                    cde.Set();
                }
            };
            return(cde);
        }
Beispiel #34
0
        public static NodeConnectionInfo GetLeaderNode(this RaftEngine engine, int waitTimeoutInSeconds = 0)
        {
            if (waitTimeoutInSeconds > 0)
            {
                engine.WaitForLeader(waitTimeoutInSeconds * 1000);
            }

            var leader = engine.CurrentLeader;

            if (leader == null)
            {
                if (waitTimeoutInSeconds > 0)
                {
                    throw new InvalidOperationException($"No leader. Waited {waitTimeoutInSeconds} seconds.");
                }

                return(null);
            }


            return(engine.CurrentTopology.AllNodes.Single(x => x.Name == leader));
        }
Beispiel #35
0
        public void UpdateTermTo(RaftEngine engine, long term)
        {
            using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
            {
                var metadata = tx.ReadTree(MetadataTreeName);

                metadata.Add("current-term", EndianBitConverter.Little.GetBytes(term));
                metadata.Add("voted-for", new byte[0]);                                  // clearing who we voted for
                metadata.Add("voted-for-term", EndianBitConverter.Little.GetBytes(-1L)); // clearing who we voted for

                VotedFor     = null;
                VotedForTerm = -1;
                CurrentTerm  = term;

                if (engine != null)
                {
                    engine.OnNewTerm(term);
                }

                tx.Commit();
            }
        }
Beispiel #36
0
        public void Follower_on_timeout_should_become_candidate()
        {
            var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();

            storageEnvironmentOptions.OwnsPagers = false;

            var nodeOptions = new RaftEngineOptions(new NodeConnectionInfo {
                Name = "real"
            }, storageEnvironmentOptions, _inMemoryTransportHub.CreateTransportFor("real"), new DictionaryStateMachine());

            PersistentState.SetTopologyExplicitly(nodeOptions,
                                                  new Topology(
                                                      new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
                                                      new[]
            {
                new NodeConnectionInfo {
                    Name = "real"
                }, new NodeConnectionInfo {
                    Name = "u2"
                }, new NodeConnectionInfo {
                    Name = "pj"
                },
            }, new NodeConnectionInfo[0], new NodeConnectionInfo[0]), throwIfTopologyExists: true);
            storageEnvironmentOptions.OwnsPagers = true;

            using (var node = new RaftEngine(nodeOptions))
            {
                var timeoutEvent = new ManualResetEventSlim();
                node.StateTimeout += timeoutEvent.Set;

                ForceTimeout("real");

                timeoutEvent.Wait();
                Assert.Equal(RaftEngineState.Candidate, node.State);
            }
        }
Beispiel #37
0
        public HttpTransportPingTest()
        {
            _node1Transport = new HttpTransport("node1");

            var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
            var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine())
                {
                    ElectionTimeout = 60 * 1000,
                    HeartbeatTimeout = 10 * 1000
                };
            _raftEngine = new RaftEngine(engineOptions);

            _server = WebApp.Start(new StartOptions
            {
                Urls = { "http://+:9079/" }
            }, builder =>
            {
                var httpConfiguration = new HttpConfiguration();
                RaftWebApiConfig.Load();
                httpConfiguration.MapHttpAttributeRoutes();
                httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus;
                builder.UseWebApi(httpConfiguration);
            });
        }
		public void Follower_on_timeout_should_become_candidate()
		{
			var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
			storageEnvironmentOptions.OwnsPagers = false;

			var nodeOptions = new RaftEngineOptions(new NodeConnectionInfo { Name = "real" }, storageEnvironmentOptions, _inMemoryTransportHub.CreateTransportFor("real"), new DictionaryStateMachine());

			PersistentState.SetTopologyExplicitly(nodeOptions,
				new Topology(
					new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
					new[]
					{
						new NodeConnectionInfo {Name = "real"}, new NodeConnectionInfo {Name = "u2"}, new NodeConnectionInfo {Name = "pj"},
					}, new NodeConnectionInfo[0], new NodeConnectionInfo[0]), throwIfTopologyExists: true);
			storageEnvironmentOptions.OwnsPagers = true;

			using (var node = new RaftEngine(nodeOptions))
			{
				var timeoutEvent = new ManualResetEventSlim();
				node.StateTimeout += timeoutEvent.Set;

				ForceTimeout("real");

				timeoutEvent.Wait();
				Assert.Equal(RaftEngineState.Candidate, node.State);
			}
		}
Beispiel #39
0
		protected ManualResetEventSlim WaitForSnapshot(RaftEngine node)
		{
			var cde = new ManualResetEventSlim();
			node.CreatedSnapshot += cde.Set;
			return cde;
		}
Beispiel #40
0
		protected ManualResetEventSlim WaitForSnapshotInstallation(RaftEngine node)
		{
			var cde = new ManualResetEventSlim();
			node.SnapshotInstalled += cde.Set;
			return cde;
		}
Beispiel #41
0
		public void UpdateTermTo(RaftEngine engine, long term)
		{
			using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
			{
				var metadata = tx.ReadTree(MetadataTreeName);

				metadata.Add("current-term", EndianBitConverter.Little.GetBytes(term));
				metadata.Add("voted-for", new byte[0]); // clearing who we voted for
				metadata.Add("voted-for-term", EndianBitConverter.Little.GetBytes(-1L)); // clearing who we voted for

				VotedFor = null;
				VotedForTerm = -1;
				CurrentTerm = term;

				if (engine != null)
					engine.OnNewTerm(term);

				tx.Commit();
			}
		}
Beispiel #42
0
 public ClusterManager(RaftEngine engine)
 {
     Engine = engine;
     Client = new ClusterManagementHttpClient(engine);
 }
Beispiel #43
0
		protected RaftEngine CreateNetworkAndGetLeader(int nodeCount, int messageTimeout = -1)
		{
			var leaderIndex = new Random().Next(0, nodeCount);
			if (messageTimeout == -1)
				messageTimeout = Debugger.IsAttached ? 3 * 1000 : 500;
			var nodeNames = new string[nodeCount];
			for (int i = 0; i < nodeCount; i++)
			{
				nodeNames[i] = "node" + i;
			}

			WriteLine("{0} selected as seed", nodeNames[leaderIndex]);
			var allNodesFinishedJoining = new ManualResetEventSlim();
			for (int index = 0; index < nodeNames.Length; index++)
			{
				var nodeName = nodeNames[index];
				var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
				storageEnvironmentOptions.OwnsPagers = false;
				var options = CreateNodeOptions(nodeName, messageTimeout, storageEnvironmentOptions, nodeNames);
				if (leaderIndex == index)
				{
					PersistentState.ClusterBootstrap(options);
				}
				storageEnvironmentOptions.OwnsPagers = true;
				var engine = new RaftEngine(options);
				_nodes.Add(engine);
				if (leaderIndex == index)
				{
					engine.TopologyChanged += command =>
					{
						if (command.Requested.AllNodeNames.All(command.Requested.IsVoter))
						{
							allNodesFinishedJoining.Set();
						}
					};
					for (int i = 0; i < nodeNames.Length; i++)
					{
						if (i == leaderIndex)
							continue;
						Assert.True(engine.AddToClusterAsync(new NodeConnectionInfo { Name = nodeNames[i] }).Wait(3000));
					}
				}
			}
			if (nodeCount == 1)
				allNodesFinishedJoining.Set();
			Assert.True(allNodesFinishedJoining.Wait(5000 * nodeCount));

			var raftEngine = _nodes[leaderIndex];


			var transport = (InMemoryTransportHub.InMemoryTransport)_inMemoryTransportHub.CreateTransportFor(raftEngine.Name);
			transport.ForceTimeout();
			Assert.True(_nodes[leaderIndex].WaitForLeader());
			var leader = _nodes.FirstOrDefault(x => x.State == RaftEngineState.Leader);
			Assert.NotNull(leader);

			return _nodes[leaderIndex];
		}
Beispiel #44
0
		protected RaftEngine NewNodeFor(RaftEngine leader)
		{
			var raftEngine = new RaftEngine(CreateNodeOptions("node" + _nodes.Count, leader.Options.ElectionTimeout, StorageEnvironmentOptions.CreateMemoryOnly()));
			_nodes.Add(raftEngine);
			return raftEngine;
		}
Beispiel #45
0
 public SnapshotInstallationStateBehavior(RaftEngine engine) : base(engine)
 {
     _random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
     Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
 }
Beispiel #46
0
 public ClusterManagementHttpClient(RaftEngine raftEngine)
 {
     this.raftEngine = raftEngine;
 }
Beispiel #47
0
		public void AppendToLog(RaftEngine engine, IEnumerable<LogEntry> entries, long removeAllAfter)
		{
			using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
			{
				var logs = tx.ReadTree(LogsTreeName);
				var terms = tx.ReadTree(EntryTermsTreeName);
				var metadata = tx.ReadTree(MetadataTreeName);
				var changed = ReadIsTopologyChanged(metadata);
				if (changed > removeAllAfter)
				{
					// need to reset the topology
					var current = metadata.Read("current-topology");
					var prevTopologyStr = current == null ? "{}" : current.Reader.ToStringValue();
					metadata.Add("current-topology", prevTopologyStr);
					metadata.Add("current-topology-index", EndianBitConverter.Little.GetBytes(0L));
					var prevTopology = JsonConvert.DeserializeObject<Topology>(prevTopologyStr);
					engine.RevertTopologyTo(prevTopology);
				}

				using (var it = logs.Iterate())
				{
					if (it.Seek(new Slice(EndianBitConverter.Big.GetBytes(removeAllAfter))) &&
						it.MoveNext())
					{
						do
						{
							terms.Delete(it.CurrentKey);
						} while (it.DeleteCurrentAndMoveNext());
					}
				}

				foreach (var entry in entries)
				{
					var key = new Slice(EndianBitConverter.Big.GetBytes(entry.Index));
					logs.Add(key, entry.Data);
					terms.Add(key, new Slice(BitConverter.GetBytes(entry.Term)));
				}

				tx.Commit();
			}
		}
		public SnapshotInstallationStateBehavior(RaftEngine engine) : base(engine)
		{
			_random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks));
			Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout);
		}
Beispiel #49
0
		protected void RestartAllNodes()
		{
			foreach (var raftEngine in _nodes)
			{
				raftEngine.Options.StorageOptions.OwnsPagers = false;
				raftEngine.Dispose();
			}
			for (int i = 0; i < _nodes.Count; i++)
			{
				_nodes[i] = new RaftEngine(_nodes[i].Options);
			}
		}