public RaftEngineOptions(NodeConnectionInfo connection, StorageEnvironmentOptions storageOptions, ITransport transport, IRaftStateMachine stateMachine) { if (connection == null) throw new ArgumentNullException("connection"); if (String.IsNullOrWhiteSpace(connection.Name)) throw new ArgumentNullException("connection.Name"); if (storageOptions == null) throw new ArgumentNullException("storageOptions"); if (transport == null) throw new ArgumentNullException("transport"); if (stateMachine == null) throw new ArgumentNullException("stateMachine"); SelfConnection = connection; StorageOptions = storageOptions; Transport = transport; StateMachine = stateMachine; ElectionTimeout = 1200; HeartbeatTimeout = 300; Stopwatch = new Stopwatch(); MaxLogLengthBeforeCompaction = 32 * 1024; MaxStepDownDrainTime = TimeSpan.FromSeconds(15); MaxEntriesPerRequest = 256; }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("append entries to " + dest, async () => { var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}", req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId); var httpResponseMessage = await client.PostAsync(requestUri,new EntriesContent(req.Entries)); var reply = await httpResponseMessage.Content.ReadAsStringAsync(); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var appendEntriesResponse = JsonConvert.DeserializeObject<AppendEntriesResponse>(reply); SendToSelf(appendEntriesResponse); }); } }
public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action<Stream> streamWriter) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("install snapshot to " + dest, async () => { var requestUri = string.Format("raft/installSnapshot?term={0}&=lastIncludedIndex={1}&lastIncludedTerm={2}&from={3}&topology={4}&clusterTopologyId={5}", req.Term, req.LastIncludedIndex, req.LastIncludedTerm, req.From, Uri.EscapeDataString(JsonConvert.SerializeObject(req.Topology)), req.ClusterTopologyId); var httpResponseMessage = await client.PostAsync(requestUri, new SnapshotContent(streamWriter)); var reply = await httpResponseMessage.Content.ReadAsStringAsync(); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error installing snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var installSnapshotResponse = JsonConvert.DeserializeObject<InstallSnapshotResponse>(reply); SendToSelf(installSnapshotResponse); }); } }
public void CanAskIfCanInstallSnapshot() { using (var node2Transport = new HttpTransport("node2")) { var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; node2Transport.Send(node1, new CanInstallSnapshotRequest { From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), Term = 2, Index = 3, }); MessageContext context; var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); var msg = (CanInstallSnapshotResponse)context.Message; Assert.True(msg.Success); } }
public void CanSendRequestVotesAndGetReply() { using (var node2Transport = new HttpTransport("node2")) { var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; node2Transport.Send(node1, new RequestVoteRequest { TrialOnly = true, From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), Term = 3, LastLogIndex = 2, LastLogTerm = 2, }); MessageContext context; var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); Assert.True(context.Message is RequestVoteResponse); } }
public HttpTransportPingTest() { _node1Transport = new HttpTransport("node1"); var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine()) { ElectionTimeout = 60 * 1000, HeartbeatTimeout = 10 * 1000 }; _raftEngine = new RaftEngine(engineOptions); _server = WebApp.Start(new StartOptions { Urls = { "http://+:9079/" } }, builder => { var httpConfiguration = new HttpConfiguration(); RaftWebApiConfig.Load(); httpConfiguration.MapHttpAttributeRoutes(); httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus; builder.UseWebApi(httpConfiguration); }); }
internal SecuredAuthenticator GetAuthenticator(NodeConnectionInfo info) { return(_securedAuthenticatorCache.GetOrAdd(info.Name, _ => new SecuredAuthenticator(autoRefreshToken: false))); }
internal Task <Action <HttpClient> > HandleForbiddenResponseAsync(HttpResponseMessage forbiddenResponse, NodeConnectionInfo nodeConnection) { if (nodeConnection.ApiKey == null) { AssertForbiddenCredentialSupportWindowsAuth(forbiddenResponse, nodeConnection); return(null); } return(null); }
private void AssertForbiddenCredentialSupportWindowsAuth(HttpResponseMessage response, NodeConnectionInfo nodeConnection) { if (nodeConnection.ToOperationCredentials().Credentials == null) { return; } var requiredAuth = response.Headers.GetFirstValue("Raven-Required-Auth"); if (requiredAuth == "Windows") { // we are trying to do windows auth, but we didn't get the windows auth headers throw new SecurityException( "Attempted to connect to a RavenDB Server that requires authentication using Windows credentials, but the specified server does not support Windows authentication." + Environment.NewLine + "If you are running inside IIS, make sure to enable Windows authentication."); } }
public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req) { _sender.Send(dest, req); }
public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req) { _parent.AddToQueue(this, dest.Name, req); }
public Task AddToClusterAsync(NodeConnectionInfo node, bool nonVoting = false) { if (_currentTopology.Contains(node.Name)) throw new InvalidOperationException("Node " + node.Name + " is already in the cluster"); var requestedTopology = new Topology( _currentTopology.TopologyId, _currentTopology.AllVotingNodes, nonVoting ? _currentTopology.NonVotingNodes.Union(new[] { node }) : _currentTopology.NonVotingNodes, nonVoting ? _currentTopology.PromotableNodes : _currentTopology.PromotableNodes.Union(new[] { node }) ); if (_log.IsInfoEnabled) { _log.Info("AddToClusterClusterAsync, requestedTopology: {0}", requestedTopology); } return ModifyTopology(requestedTopology); }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { _sender.Send(dest, req); }
public void CanSendEntries() { using (var node2Transport = new HttpTransport("node2")) { var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; node2Transport.Send(node1, new AppendEntriesRequest { From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), Term = 2, PrevLogIndex = 0, PrevLogTerm = 0, LeaderCommit = 1, Entries = new LogEntry[] { new LogEntry { Term = 2, Index = 1, Data = new JsonCommandSerializer().Serialize(new DictionaryCommand.Set { Key = "a", Value = 2 }) }, } }); MessageContext context; var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); var appendEntriesResponse = (AppendEntriesResponse)context.Message; Assert.True(appendEntriesResponse.Success); Assert.Equal(2, ((DictionaryStateMachine)_raftEngine.StateMachine).Data["a"]); } }
public void CanSendTimeoutNow() { using (var node2Transport = new HttpTransport("node2")) { var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; node2Transport.Send(node1, new AppendEntriesRequest { From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), Term = 2, PrevLogIndex = 0, PrevLogTerm = 0, LeaderCommit = 1, Entries = new[] { new LogEntry { Term = 2, Index = 1, Data = new JsonCommandSerializer().Serialize(new DictionaryCommand.Set { Key = "a", Value = 2 }) }, } }); MessageContext context; var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); Assert.True(((AppendEntriesResponse)context.Message).Success); var mres = new ManualResetEventSlim(); _raftEngine.StateChanged += state => { if (state == RaftEngineState.CandidateByRequest) mres.Set(); }; node2Transport.Send(node1, new TimeoutNowRequest { Term = 4, From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), }); gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); Assert.True(context.Message is NothingToDo); Assert.True(mres.Wait(_timeout)); } }
public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("disconnect " + dest, async () => { var message = await client.GetAsync(string.Format("raft/disconnectFromCluster?term={0}&from={1}&clusterTopologyId={2}", req.Term, req.From, req.ClusterTopologyId)); var reply = await message.Content.ReadAsStringAsync(); if (message.IsSuccessStatusCode == false) { _log.Warn("Error sending disconnecton notification to {0}. Status: {1}\r\n{2}", dest.Name, message.StatusCode, message, reply); return; } SendToSelf(new NothingToDo()); }); } }
public void CanInstallSnapshot() { using (var node2Transport = new HttpTransport("node2")) { var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; node2Transport.Send(node1, new CanInstallSnapshotRequest { From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), Term = 2, Index = 3, }); MessageContext context; var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); Assert.True(context.Message is CanInstallSnapshotResponse); node2Transport.Stream(node1, new InstallSnapshotRequest { From = "node2", ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), Term = 2, Topology = new Topology(new Guid("355a589b-cadc-463d-a515-5add2ea47205")), LastIncludedIndex = 2, LastIncludedTerm = 2, }, stream => { var streamWriter = new StreamWriter(stream); var data = new Dictionary<string, int> { { "a", 2 } }; new JsonSerializer().Serialize(streamWriter, data); streamWriter.Flush(); }); gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); Assert.True(gotIt); var appendEntriesResponse = (InstallSnapshotResponse)context.Message; Assert.True(appendEntriesResponse.Success); Assert.Equal(2, ((DictionaryStateMachine)_raftEngine.StateMachine).Data["a"]); } }
private ReturnToQueue GetConnection(NodeConnectionInfo info, out HttpClient result) { var connectionQueue = _httpClientsCache.GetOrAdd(info.Name, _ => new ConcurrentQueue<HttpClient>()); if (connectionQueue.TryDequeue(out result) == false) { result = new HttpClient { BaseAddress = info.Uri }; } return new ReturnToQueue(result, connectionQueue); }
public void Send(NodeConnectionInfo dest, RequestVoteRequest req) { _parent.AddToQueue(this, dest.Name, req); }
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) { _parent.AddToQueue(this, dest.Name, req); }
public HttpCacheKey(NodeConnectionInfo nodeConnection, TimeSpan timeout) { NodeConnection = nodeConnection; Timeout = timeout; }
public void Send(NodeConnectionInfo dest, TimeoutNowRequest req) { _sender.Send(dest, req); }
public Task RemoveFromClusterAsync(NodeConnectionInfo node) { if (_currentTopology.Contains(node.Name) == false) throw new InvalidOperationException("Node " + node + " was not found in the cluster"); if (string.Equals(node.Name, Name, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException("You cannot remove the current node from the cluster, step down this node and then remove it from the new leader"); var requestedTopology = new Topology( _currentTopology.TopologyId, _currentTopology.AllVotingNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false), _currentTopology.NonVotingNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false), _currentTopology.PromotableNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false) ); if (_log.IsInfoEnabled) { _log.Info("RemoveFromClusterAsync, requestedTopology: {0}", requestedTopology); } return ModifyTopology(requestedTopology); }
private void SendEntriesToPeer(NodeConnectionInfo peer) { LogEntry prevLogEntry; LogEntry[] entries; var nextIndex = _nextIndexes.GetOrAdd(peer.Name, -1); //new peer's index starts at, we use -1 as a sentital value if (Engine.StateMachine.SupportSnapshots && nextIndex != -1) { var snapshotIndex = Engine.PersistentState.GetLastSnapshotIndex(); if (snapshotIndex != null && nextIndex < snapshotIndex) { if (_snapshotsPendingInstallation.ContainsKey(peer.Name)) return; using (var snapshotWriter = Engine.StateMachine.GetSnapshotWriter()) { Engine.Transport.Send(peer, new CanInstallSnapshotRequest { From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, Index = snapshotWriter.Index, Term = snapshotWriter.Term, }); } return; } } if (nextIndex == -1) nextIndex = 0; try { entries = Engine.PersistentState.LogEntriesAfter(nextIndex) .Take(Engine.Options.MaxEntriesPerRequest) .ToArray(); prevLogEntry = entries.Length == 0 ? Engine.PersistentState.LastLogEntry() : Engine.PersistentState.GetLogEntry(entries[0].Index - 1); } catch (Exception e) { _log.Error("Error while fetching entries from persistent state.", e); throw; } if (_log.IsDebugEnabled) { _log.Debug("Sending {0:#,#;;0} entries to {1} (PrevLogEntry: Term = {2} Index = {3}).", entries.Length, peer, prevLogEntry.Term, prevLogEntry.Index); } var aer = new AppendEntriesRequest { Entries = entries, LeaderCommit = Engine.CommitIndex, PrevLogIndex = prevLogEntry.Index, PrevLogTerm = prevLogEntry.Term, Term = Engine.PersistentState.CurrentTerm, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }; Engine.Transport.Send(peer, aer); Engine.OnEntriesAppended(entries); }
public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req) { _sender.Send(dest, req); }
public void Send(NodeConnectionInfo dest, RequestVoteRequest req) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("request vote from " + dest, async () => { var requestUri = string.Format("raft/requestVote?term={0}&lastLogIndex={1}&lastLogTerm={2}&trialOnly={3}&forcedElection={4}&from={5}&clusterTopologyId={6}", req.Term, req.LastLogIndex, req.LastLogTerm, req.TrialOnly, req.ForcedElection, req.From, req.ClusterTopologyId); var httpResponseMessage = await client.GetAsync(requestUri); var reply = await httpResponseMessage.Content.ReadAsStringAsync(); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error requesting vote from {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var requestVoteResponse = JsonConvert.DeserializeObject<RequestVoteResponse>(reply); SendToSelf(requestVoteResponse); }); } }
public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action<Stream> streamWriter) { _sender.Stream(dest, req, streamWriter); }
private void AssertUnauthorizedCredentialSupportWindowsAuth(HttpResponseMessage response, NodeConnectionInfo nodeConnectionInfo) { if (nodeConnectionInfo.Username == null) { return; } var authHeaders = response.Headers.WwwAuthenticate.FirstOrDefault(); if (authHeaders == null || (authHeaders.ToString().Contains("NTLM") == false && authHeaders.ToString().Contains("Negotiate") == false)) { // we are trying to do windows auth, but we didn't get the windows auth headers throw new SecurityException( "Attempted to connect to a RavenDB Server that requires authentication using Windows credentials," + Environment.NewLine + " but either wrong credentials where entered or the specified server does not support Windows authentication." + Environment.NewLine + "If you are running inside IIS, make sure to enable Windows authentication."); } }
public void Send(NodeConnectionInfo dest, RequestVoteRequest req) { _sender.Send(dest, req); }
public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req) { _parent.AddToQueue(this, dest.Name, req); }
protected bool Equals(NodeConnectionInfo other) { return(Equals(Uri, other.Uri) && string.Equals(Name, other.Name) && string.Equals(Username, other.Username) && string.Equals(Password, other.Password) && string.Equals(Domain, other.Domain) && string.Equals(ApiKey, other.ApiKey)); }
internal async Task <Action <HttpClient> > HandleUnauthorizedResponseAsync(HttpResponseMessage unauthorizedResponse, NodeConnectionInfo nodeConnectionInfo) { var oauthSource = unauthorizedResponse.Headers.GetFirstValue("OAuth-Source"); if (nodeConnectionInfo.ApiKey == null) { AssertUnauthorizedCredentialSupportWindowsAuth(unauthorizedResponse, nodeConnectionInfo); return(null); } if (string.IsNullOrEmpty(oauthSource)) { oauthSource = nodeConnectionInfo.Uri.AbsoluteUri + "/OAuth/API-Key"; } return(await GetAuthenticator(nodeConnectionInfo).DoOAuthRequestAsync(nodeConnectionInfo.Uri.AbsoluteUri, oauthSource, nodeConnectionInfo.ApiKey).ConfigureAwait(false)); }
private void SendSnapshotToPeer(NodeConnectionInfo peer) { try { var sp = Stopwatch.StartNew(); using (var snapshotWriter = Engine.StateMachine.GetSnapshotWriter()) { _log.Info("Streaming snapshot to {0} - term {1}, index {2}", peer, snapshotWriter.Term, snapshotWriter.Index); Engine.Transport.Stream(peer, new InstallSnapshotRequest { Term = Engine.PersistentState.CurrentTerm, LastIncludedIndex = snapshotWriter.Index, LastIncludedTerm = snapshotWriter.Term, From = Engine.Name, ClusterTopologyId = Engine.CurrentTopology.TopologyId, }, stream => snapshotWriter.WriteSnapshot(stream)); _log.Info("Finished snapshot streaming -> to {0} - term {1}, index {2} in {3}", peer, snapshotWriter.Index, snapshotWriter.Term, sp.Elapsed); } } catch (Exception e) { _log.Error("Failed to send snapshot to " + peer, e); } }
public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req) { HttpClient client; using (GetConnection(dest, out client)) { LogStatus("can install snapshot to " + dest, async () => { var requestUri = string.Format("raft/canInstallSnapshot?term={0}&=index{1}&from={2}&clusterTopologyId={3}", req.Term, req.Index, req.From, req.ClusterTopologyId); var httpResponseMessage = await client.GetAsync(requestUri); var reply = await httpResponseMessage.Content.ReadAsStringAsync(); if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) { _log.Warn("Error checking if can install snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); return; } var canInstallSnapshotResponse = JsonConvert.DeserializeObject<CanInstallSnapshotResponse>(reply); SendToSelf(canInstallSnapshotResponse); }); } }