public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action<Stream> streamWriter)
{
HttpClient client;
using (GetConnection(dest, out client))
{
LogStatus("install snapshot to " + dest, async () =>
{
var requestUri =
string.Format("raft/installSnapshot?term={0}&=lastIncludedIndex={1}&lastIncludedTerm={2}&from={3}&topology={4}&clusterTopologyId={5}",
req.Term, req.LastIncludedIndex, req.LastIncludedTerm, req.From, Uri.EscapeDataString(JsonConvert.SerializeObject(req.Topology)), req.ClusterTopologyId);
var httpResponseMessage = await client.PostAsync(requestUri, new SnapshotContent(streamWriter));
var reply = await httpResponseMessage.Content.ReadAsStringAsync();
if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable)
{
_log.Warn("Error installing snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply);
return;
}
var installSnapshotResponse = JsonConvert.DeserializeObject<InstallSnapshotResponse>(reply);
SendToSelf(installSnapshotResponse);
});
}
}
public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action<Stream> streamWriter)
{
_sender.Stream(dest, req, streamWriter);
}
public virtual InstallSnapshotResponse Handle(MessageContext context, InstallSnapshotRequest req, Stream stream)
{
if (FromOurTopology(req) == false)
{
_log.Info("Got an install snapshot message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId);
return new InstallSnapshotResponse
{
Success = false,
Message = "Cannot accept message from outside my topology. My cluster topology id is: " + Engine.CurrentTopology.TopologyId,
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index
};
}
stream.Dispose();
return new InstallSnapshotResponse
{
Success = false,
Message = "Cannot install snapshot from state " + State,
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index
};
}
public override InstallSnapshotResponse Handle(MessageContext context, InstallSnapshotRequest req, Stream stream)
{
if (_installingSnapshot != null)
{
return new InstallSnapshotResponse
{
Success = false,
Message = "Cannot install snapshot because we are already installing a snapshot",
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index
};
}
if (FromOurTopology(req) == false)
{
_log.Info("Got an install snapshot message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId);
return new InstallSnapshotResponse
{
Success = false,
Message = "Cannot install snapshot because the cluster topology id doesn't match, mine is: " + Engine.CurrentTopology.TopologyId,
CurrentTerm = Engine.PersistentState.CurrentTerm,
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
LastLogIndex = Engine.PersistentState.LastLogEntry().Index
};
}
var lastLogEntry = Engine.PersistentState.LastLogEntry();
if (req.Term < lastLogEntry.Term || req.LastIncludedIndex < lastLogEntry.Index)
{
stream.Dispose();
return new InstallSnapshotResponse
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
CurrentTerm = lastLogEntry.Term,
LastLogIndex = lastLogEntry.Index,
Message = string.Format("Snapshot is too old (term {0} index {1}) while we have (term {2} index {3})",
req.Term, req.LastIncludedIndex, lastLogEntry.Term, lastLogEntry.Index),
Success = false
};
}
_log.Info("Received InstallSnapshotRequest from {0} until term {1} / {2}", req.From, req.LastIncludedTerm, req.LastIncludedIndex);
Engine.OnSnapshotInstallationStarted();
// this can be a long running task
_installingSnapshot = Task.Run(() =>
{
try
{
Engine.StateMachine.ApplySnapshot(req.LastIncludedTerm, req.LastIncludedIndex, stream);
Engine.PersistentState.MarkSnapshotFor(req.LastIncludedIndex, req.LastIncludedTerm, int.MaxValue);
Engine.PersistentState.SetCurrentTopology(req.Topology, req.LastIncludedIndex);
var tcc = new TopologyChangeCommand { Requested = req.Topology };
Engine.StartTopologyChange(tcc);
Engine.CommitTopologyChange(tcc);
}
catch (Exception e)
{
_log.Warn(string.Format("Failed to install snapshot term {0} index {1}", req.LastIncludedIndex, req.LastIncludedIndex), e);
context.ExecuteInEventLoop(() =>
{
_installingSnapshot = null;
});
}
// we are doing it this way to ensure that we are single threaded
context.ExecuteInEventLoop(() =>
{
Engine.UpdateCurrentTerm(req.Term, req.From); // implicitly put us in follower state
_log.Info("Updating the commit index to the snapshot last included index of {0}", req.LastIncludedIndex);
Engine.OnSnapshotInstallationEnded(req.Term);
context.Reply(new InstallSnapshotResponse
{
From = Engine.Name,
ClusterTopologyId = Engine.CurrentTopology.TopologyId,
CurrentTerm = req.Term,
LastLogIndex = req.LastIncludedIndex,
Success = true
});
});
});
return null;
}