public RaftEngineOptions(NodeConnectionInfo connection, StorageEnvironmentOptions storageOptions, ITransport transport, IRaftStateMachine stateMachine)
{
if (connection == null) throw new ArgumentNullException("connection");
if (String.IsNullOrWhiteSpace(connection.Name)) throw new ArgumentNullException("connection.Name");
if (storageOptions == null) throw new ArgumentNullException("storageOptions");
if (transport == null) throw new ArgumentNullException("transport");
if (stateMachine == null) throw new ArgumentNullException("stateMachine");
SelfConnection = connection;
StorageOptions = storageOptions;
Transport = transport;
StateMachine = stateMachine;
ElectionTimeout = 1200;
HeartbeatTimeout = 300;
Stopwatch = new Stopwatch();
MaxLogLengthBeforeCompaction = 32 * 1024;
MaxStepDownDrainTime = TimeSpan.FromSeconds(15);
MaxEntriesPerRequest = 256;
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("append entries to " + dest, async () =>
{
var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId);
var httpResponseMessage = await client.PostAsync(requestUri,new EntriesContent(req.Entries));
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var appendEntriesResponse = JsonConvert.DeserializeObject<AppendEntriesResponse>(reply);
SendToSelf(appendEntriesResponse);
});
}
}
public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action<Stream> streamWriter)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("install snapshot to " + dest, async () =>
{
var requestUri =
string.Format("raft/installSnapshot?term={0}&=lastIncludedIndex={1}&lastIncludedTerm={2}&from={3}&topology={4}&clusterTopologyId={5}",
req.Term, req.LastIncludedIndex, req.LastIncludedTerm, req.From, Uri.EscapeDataString(JsonConvert.SerializeObject(req.Topology)), req.ClusterTopologyId);
var httpResponseMessage = await client.PostAsync(requestUri, new SnapshotContent(streamWriter));
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error installing snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var installSnapshotResponse = JsonConvert.DeserializeObject<InstallSnapshotResponse>(reply);
SendToSelf(installSnapshotResponse);
});
}
}
public void CanAskIfCanInstallSnapshot()
{
using (var node2Transport = new HttpTransport("node2"))
{
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
node2Transport.Send(node1, new CanInstallSnapshotRequest
{
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
Term = 2,
Index = 3,
});
MessageContext context;
var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
var msg = (CanInstallSnapshotResponse)context.Message;
Assert.True(msg.Success);
}
}
public void CanSendRequestVotesAndGetReply()
{
using (var node2Transport = new HttpTransport("node2"))
{
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
node2Transport.Send(node1, new RequestVoteRequest
{
TrialOnly = true,
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
Term = 3,
LastLogIndex = 2,
LastLogTerm = 2,
});
MessageContext context;
var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
Assert.True(context.Message is RequestVoteResponse);
}
}
public HttpTransportPingTest()
{
_node1Transport = new HttpTransport("node1");
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine())
{
ElectionTimeout = 60 * 1000,
HeartbeatTimeout = 10 * 1000
};
_raftEngine = new RaftEngine(engineOptions);
_server = WebApp.Start(new StartOptions
{
Urls = { "http://+:9079/" }
}, builder =>
{
var httpConfiguration = new HttpConfiguration();
RaftWebApiConfig.Load();
httpConfiguration.MapHttpAttributeRoutes();
httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus;
builder.UseWebApi(httpConfiguration);
});
}
internal SecuredAuthenticator GetAuthenticator(NodeConnectionInfo info)
{
return(_securedAuthenticatorCache.GetOrAdd(info.Name, _ => new SecuredAuthenticator(autoRefreshToken: false)));
}
internal Task <Action <HttpClient> > HandleForbiddenResponseAsync(HttpResponseMessage forbiddenResponse, NodeConnectionInfo nodeConnection)
{
if (nodeConnection.ApiKey == null)
{
AssertForbiddenCredentialSupportWindowsAuth(forbiddenResponse, nodeConnection);
return(null);
}
return(null);
}
private void AssertForbiddenCredentialSupportWindowsAuth(HttpResponseMessage response, NodeConnectionInfo nodeConnection)
{
if (nodeConnection.ToOperationCredentials().Credentials == null)
{
return;
}
var requiredAuth = response.Headers.GetFirstValue("Raven-Required-Auth");
if (requiredAuth == "Windows")
{
// we are trying to do windows auth, but we didn't get the windows auth headers
throw new SecurityException(
"Attempted to connect to a RavenDB Server that requires authentication using Windows credentials, but the specified server does not support Windows authentication." +
Environment.NewLine +
"If you are running inside IIS, make sure to enable Windows authentication.");
}
}
public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req)
{
_sender.Send(dest, req);
}
public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req)
{
_parent.AddToQueue(this, dest.Name, req);
}
public Task AddToClusterAsync(NodeConnectionInfo node, bool nonVoting = false)
{
if (_currentTopology.Contains(node.Name))
throw new InvalidOperationException("Node " + node.Name + " is already in the cluster");
var requestedTopology = new Topology(
_currentTopology.TopologyId,
_currentTopology.AllVotingNodes,
nonVoting ? _currentTopology.NonVotingNodes.Union(new[] { node }) : _currentTopology.NonVotingNodes,
nonVoting ? _currentTopology.PromotableNodes : _currentTopology.PromotableNodes.Union(new[] { node })
);
if (_log.IsInfoEnabled)
{
_log.Info("AddToClusterClusterAsync, requestedTopology: {0}", requestedTopology);
}
return ModifyTopology(requestedTopology);
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
_sender.Send(dest, req);
}
public void CanSendEntries()
{
using (var node2Transport = new HttpTransport("node2"))
{
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
node2Transport.Send(node1, new AppendEntriesRequest
{
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
Term = 2,
PrevLogIndex = 0,
PrevLogTerm = 0,
LeaderCommit = 1,
Entries = new LogEntry[]
{
new LogEntry
{
Term = 2,
Index = 1,
Data = new JsonCommandSerializer().Serialize(new DictionaryCommand.Set
{
Key = "a",
Value = 2
})
},
}
});
MessageContext context;
var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
var appendEntriesResponse = (AppendEntriesResponse)context.Message;
Assert.True(appendEntriesResponse.Success);
Assert.Equal(2, ((DictionaryStateMachine)_raftEngine.StateMachine).Data["a"]);
}
}
public void CanSendTimeoutNow()
{
using (var node2Transport = new HttpTransport("node2"))
{
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
node2Transport.Send(node1, new AppendEntriesRequest
{
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
Term = 2,
PrevLogIndex = 0,
PrevLogTerm = 0,
LeaderCommit = 1,
Entries = new[]
{
new LogEntry
{
Term = 2,
Index = 1,
Data = new JsonCommandSerializer().Serialize(new DictionaryCommand.Set
{
Key = "a",
Value = 2
})
},
}
});
MessageContext context;
var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
Assert.True(((AppendEntriesResponse)context.Message).Success);
var mres = new ManualResetEventSlim();
_raftEngine.StateChanged += state =>
{
if (state == RaftEngineState.CandidateByRequest)
mres.Set();
};
node2Transport.Send(node1, new TimeoutNowRequest
{
Term = 4,
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
});
gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
Assert.True(context.Message is NothingToDo);
Assert.True(mres.Wait(_timeout));
}
}
public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("disconnect " + dest, async () =>
{
var message = await client.GetAsync(string.Format("raft/disconnectFromCluster?term={0}&from={1}&clusterTopologyId={2}", req.Term, req.From, req.ClusterTopologyId));
var reply = await message.Content.ReadAsStringAsync();
if (message.IsSuccessStatusCode == false)
{
_log.Warn("Error sending disconnecton notification to {0}. Status: {1}\r\n{2}", dest.Name, message.StatusCode, message, reply);
return;
}
SendToSelf(new NothingToDo());
});
}
}
public void CanInstallSnapshot()
{
using (var node2Transport = new HttpTransport("node2"))
{
var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") };
node2Transport.Send(node1, new CanInstallSnapshotRequest
{
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
Term = 2,
Index = 3,
});
MessageContext context;
var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
Assert.True(context.Message is CanInstallSnapshotResponse);
node2Transport.Stream(node1, new InstallSnapshotRequest
{
From = "node2",
ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"),
Term = 2,
Topology = new Topology(new Guid("355a589b-cadc-463d-a515-5add2ea47205")),
LastIncludedIndex = 2,
LastIncludedTerm = 2,
}, stream =>
{
var streamWriter = new StreamWriter(stream);
var data = new Dictionary<string, int> { { "a", 2 } };
new JsonSerializer().Serialize(streamWriter, data);
streamWriter.Flush();
});
gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context);
Assert.True(gotIt);
var appendEntriesResponse = (InstallSnapshotResponse)context.Message;
Assert.True(appendEntriesResponse.Success);
Assert.Equal(2, ((DictionaryStateMachine)_raftEngine.StateMachine).Data["a"]);
}
}
private ReturnToQueue GetConnection(NodeConnectionInfo info, out HttpClient result)
{
var connectionQueue = _httpClientsCache.GetOrAdd(info.Name, _ => new ConcurrentQueue<HttpClient>());
if (connectionQueue.TryDequeue(out result) == false)
{
result = new HttpClient
{
BaseAddress = info.Uri
};
}
return new ReturnToQueue(result, connectionQueue);
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
_parent.AddToQueue(this, dest.Name, req);
}
public void Send(NodeConnectionInfo dest, AppendEntriesRequest req)
{
_parent.AddToQueue(this, dest.Name, req);
}
public HttpCacheKey(NodeConnectionInfo nodeConnection, TimeSpan timeout)
{
NodeConnection = nodeConnection;
Timeout = timeout;
}
public void Send(NodeConnectionInfo dest, TimeoutNowRequest req)
{
_sender.Send(dest, req);
}
public Task RemoveFromClusterAsync(NodeConnectionInfo node)
{
if (_currentTopology.Contains(node.Name) == false)
throw new InvalidOperationException("Node " + node + " was not found in the cluster");
if (string.Equals(node.Name, Name, StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException("You cannot remove the current node from the cluster, step down this node and then remove it from the new leader");
var requestedTopology = new Topology(
_currentTopology.TopologyId,
_currentTopology.AllVotingNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false),
_currentTopology.NonVotingNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false),
_currentTopology.PromotableNodes.Where(x => string.Equals(x.Name, node.Name, StringComparison.OrdinalIgnoreCase) == false)
);
if (_log.IsInfoEnabled)
{
_log.Info("RemoveFromClusterAsync, requestedTopology: {0}", requestedTopology);
}
return ModifyTopology(requestedTopology);
}
private void SendEntriesToPeer(NodeConnectionInfo peer)
{
LogEntry prevLogEntry;
LogEntry[] entries;
var nextIndex = _nextIndexes.GetOrAdd(peer.Name, -1); //new peer's index starts at, we use -1 as a sentital value
if (Engine.StateMachine.SupportSnapshots && nextIndex != -1)
{
var snapshotIndex = Engine.PersistentState.GetLastSnapshotIndex();
if (snapshotIndex != null && nextIndex < snapshotIndex)
{
if (_snapshotsPendingInstallation.ContainsKey(peer.Name))
return;
using (var snapshotWriter = Engine.StateMachine.GetSnapshotWriter())
{
Engine.Transport.Send(peer, new CanInstallSnapshotRequest
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
Index = snapshotWriter.Index,
Term = snapshotWriter.Term,
});
}
return;
}
}
if (nextIndex == -1)
nextIndex = 0;
try
{
entries = Engine.PersistentState.LogEntriesAfter(nextIndex)
.Take(Engine.Options.MaxEntriesPerRequest)
.ToArray();
prevLogEntry = entries.Length == 0
? Engine.PersistentState.LastLogEntry()
: Engine.PersistentState.GetLogEntry(entries[0].Index - 1);
}
catch (Exception e)
{
_log.Error("Error while fetching entries from persistent state.", e);
throw;
}
if (_log.IsDebugEnabled)
{
_log.Debug("Sending {0:#,#;;0} entries to {1} (PrevLogEntry: Term = {2} Index = {3}).", entries.Length, peer, prevLogEntry.Term, prevLogEntry.Index);
}
var aer = new AppendEntriesRequest
{
Entries = entries,
LeaderCommit = Engine.CommitIndex,
PrevLogIndex = prevLogEntry.Index,
PrevLogTerm = prevLogEntry.Term,
Term = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
};
Engine.Transport.Send(peer, aer);
Engine.OnEntriesAppended(entries);
}
public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req)
{
_sender.Send(dest, req);
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("request vote from " + dest, async () =>
{
var requestUri = string.Format("raft/requestVote?term={0}&lastLogIndex={1}&lastLogTerm={2}&trialOnly={3}&forcedElection={4}&from={5}&clusterTopologyId={6}",
req.Term, req.LastLogIndex, req.LastLogTerm, req.TrialOnly, req.ForcedElection, req.From, req.ClusterTopologyId);
var httpResponseMessage = await client.GetAsync(requestUri);
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error requesting vote from {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var requestVoteResponse = JsonConvert.DeserializeObject<RequestVoteResponse>(reply);
SendToSelf(requestVoteResponse);
});
}
}
public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action<Stream> streamWriter)
{
_sender.Stream(dest, req, streamWriter);
}
private void AssertUnauthorizedCredentialSupportWindowsAuth(HttpResponseMessage response, NodeConnectionInfo nodeConnectionInfo)
{
if (nodeConnectionInfo.Username == null)
{
return;
}
var authHeaders = response.Headers.WwwAuthenticate.FirstOrDefault();
if (authHeaders == null || (authHeaders.ToString().Contains("NTLM") == false && authHeaders.ToString().Contains("Negotiate") == false))
{
// we are trying to do windows auth, but we didn't get the windows auth headers
throw new SecurityException(
"Attempted to connect to a RavenDB Server that requires authentication using Windows credentials," + Environment.NewLine
+ " but either wrong credentials where entered or the specified server does not support Windows authentication." +
Environment.NewLine +
"If you are running inside IIS, make sure to enable Windows authentication.");
}
}
public void Send(NodeConnectionInfo dest, RequestVoteRequest req)
{
_sender.Send(dest, req);
}
public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req)
{
_parent.AddToQueue(this, dest.Name, req);
}
protected bool Equals(NodeConnectionInfo other)
{
return(Equals(Uri, other.Uri) && string.Equals(Name, other.Name) && string.Equals(Username, other.Username) && string.Equals(Password, other.Password) && string.Equals(Domain, other.Domain) && string.Equals(ApiKey, other.ApiKey));
}
internal async Task <Action <HttpClient> > HandleUnauthorizedResponseAsync(HttpResponseMessage unauthorizedResponse, NodeConnectionInfo nodeConnectionInfo)
{
var oauthSource = unauthorizedResponse.Headers.GetFirstValue("OAuth-Source");
if (nodeConnectionInfo.ApiKey == null)
{
AssertUnauthorizedCredentialSupportWindowsAuth(unauthorizedResponse, nodeConnectionInfo);
return(null);
}
if (string.IsNullOrEmpty(oauthSource))
{
oauthSource = nodeConnectionInfo.Uri.AbsoluteUri + "/OAuth/API-Key";
}
return(await GetAuthenticator(nodeConnectionInfo).DoOAuthRequestAsync(nodeConnectionInfo.Uri.AbsoluteUri, oauthSource, nodeConnectionInfo.ApiKey).ConfigureAwait(false));
}
private void SendSnapshotToPeer(NodeConnectionInfo peer)
{
try
{
var sp = Stopwatch.StartNew();
using (var snapshotWriter = Engine.StateMachine.GetSnapshotWriter())
{
_log.Info("Streaming snapshot to {0} - term {1}, index {2}", peer,
snapshotWriter.Term,
snapshotWriter.Index);
Engine.Transport.Stream(peer, new InstallSnapshotRequest
{
Term = Engine.PersistentState.CurrentTerm,
LastIncludedIndex = snapshotWriter.Index,
LastIncludedTerm = snapshotWriter.Term,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
}, stream => snapshotWriter.WriteSnapshot(stream));
_log.Info("Finished snapshot streaming -> to {0} - term {1}, index {2} in {3}", peer, snapshotWriter.Index,
snapshotWriter.Term, sp.Elapsed);
}
}
catch (Exception e)
{
_log.Error("Failed to send snapshot to " + peer, e);
}
}
public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("can install snapshot to " + dest, async () =>
{
var requestUri = string.Format("raft/canInstallSnapshot?term={0}&=index{1}&from={2}&clusterTopologyId={3}", req.Term, req.Index,
req.From, req.ClusterTopologyId);
var httpResponseMessage = await client.GetAsync(requestUri);
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error checking if can install snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var canInstallSnapshotResponse = JsonConvert.DeserializeObject<CanInstallSnapshotResponse>(reply);
SendToSelf(canInstallSnapshotResponse);
});
}
}