private void HandleFailure(ProcedureCallBatch request, IResponseHandler rh, bool voting)
{
Dictionary <string, bool> r0 = new Dictionary <string, bool>();
foreach (Call r1 in request.Batch)
{
r0.Add(r1.ID, false);
}
// (State state, bool b) s = CurrentState.Instance.Get_State;
// bool call = voting && s.state == State.CANDIDATE;
// Activate the responsehandler
// if (call && (StateLog.Instance.Nodes.GetHeartbeat(ID) == Params.MAX_RETRIES || CurrentState.Instance.ReceviedVote ))
rh.OnResponse(ID, request, Encoder.SerializeResponse(new Response()
{
Status = r0,
Completed = null,
Data = null
}));
Logger.Write(Logger.Tag.WARN, "Unable to connect to [node:" + ID + "][retries:" + StateLog.Instance.Nodes.GetHeartbeat(ID) + "]");
StateLog.Instance.Nodes.AccessHeartbeat(ID, (hb) => { hb.Increment(); });
_notified = true;
// if (!call && StateLog.Instance.Nodes.GetHeartbeat(ID) < Params.MAX_RETRIES) StartClient(request, rh);
}
public void ReceiveBroadcast(Broadcast broadcast)
{
if (broadcast.Self.ID == Params.ID || StateLog.Instance.Nodes.ContainsPeer(broadcast.Self.ID))
{
return;
}
CurrentState.Instance.SetCandidateResolve(false);
Logger.Write(Logger.Tag.INFO, "Received broadcast from [node:" + broadcast.Self.ID + "]");
if (!CurrentState.Instance.IsLeader)
{
CurrentState.Instance.Timer.Reset();
CurrentState.Instance.CancelState();
}
StateLog.Instance.Nodes.AddIfNew(broadcast.Self);
Call request = new Call()
{
ID = Utils.GenerateUUID(),
Data = Encoder.EncodeNode(broadcast.Self),
Type = Call._Type.NODE_ADD
};
Task[] tasks;
StateLog.Instance.Nodes.ForEachAsync((p0) => {
ProcedureCallBatch rb = new ProcedureCallBatch()
{
Batch = new List <Call> {
request
},
Completed = StateLog.Instance.Leader_GetActionsCompleted(p0.Client.ID),
Sender = Credentials.Self
};
rb.Batch.AddRange(StateLog.Instance.GetBatchesBehind(p0.Client.ID));
StateLog.Instance.Nodes.ForEachPeer((p1) => {
if (p1.Client.ID != p0.Client.ID)
{
rb.Batch.Add(new Call()
{
ID = Utils.GenerateUUID(),
Data = Encoder.EncodeNode(p1.AsNode),
Type = Call._Type.NODE_ADD
});
}
});
p0.Client.StartClient(rb, new DefaultHandler());
}, out tasks);
Parallel.ForEach <Task>(tasks, (t) => { t.Start(); });
Task.WaitAll(tasks);
}
public void OnResponse(string receiverID, ProcedureCallBatch sender, string response)
{
if (sender.Completed != null)
{
foreach (string a in sender.Completed)
{
StateLog.Instance.Leader_RemoveActionsCompleted(receiverID, a);
}
}
if (response == null || receiverID == null)
{
return;
}
Response r0 = Encoder.DecodeResponse(response);
if (r0 == null || r0.Status == null)
{
return;
}
try
{
if (r0.Completed != null)
{
r0.Completed.ForEach((a) => {
StateLog.Instance.Leader_AddActionCompleted(a, receiverID);
});
}
} catch (Exception e) { Logger.Write(Logger.Tag.ERROR, e.ToString()); }
sender.Batch.ForEach((r) => {
if (r0.Status.ContainsKey(r.ID) && r0.Status[r.ID])
{
StateLog.Instance.Nodes.AccessHeartbeat(receiverID, (i) => i.Reset());
StateLog.Instance.RemoveBatchBehind(receiverID, r);
}
else
{
StateLog.Instance.AddToNodeBehind(receiverID, r);
}
});
Node node;
bool nb = StateLog.Instance.Nodes.TryGetNode(receiverID, out node);
if (nb)
{
node.HIEVar = r0.HIEVar;
}
}
/// <summary>
/// Establishes connection to the remote host, sends the request an activates OnResponse on the requesthandler
/// </summary>
public void StartClient(ProcedureCallBatch request, IResponseHandler rh, bool voting = false)
{
// Connect to a remote device.
_received.Clear();
remoteEP = new IPEndPoint(IPAddress.Parse(Host), Port);
sender = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
try {
_notified = false;
sender.LingerState = _linger;
// Connect the socket to the remote endpoint. Catch any errors.
try {
IAsyncResult result = sender.BeginConnect(remoteEP, null, null);
bool success = result.AsyncWaitHandle.WaitOne(Params.TIMEOUT, true);
if (success && sender.Connected)
{
sender.EndConnect(result);
// Encode the data string into a byte array.
byte[] msg = Encoder.EncodeProcedureCallBatch(request);
// Send the data through the socket.
int bytesSent = sender.Send(msg);
// Data buffer for incoming data.
byte[] bytes = new byte[2048 * Params.BATCH_SIZE]; // hope thats enough
// Receive the response from the remote device.
int bytesRec = sender.Receive(bytes);
if (bytesRec < 1)
{
HandleFailure(request, rh, voting);
}
rh.OnResponse(ID, request, Encoding.ASCII.GetString(bytes));
_notified = true;
}
} catch (ArgumentNullException ane) {
Logger.Write(Logger.Tag.ERROR, string.Format("ArgumentNullException : {0}", ane.ToString()));
} catch (SocketException se) {
Logger.Write(Logger.Tag.ERROR, string.Format("ArgumentNullException : {0}", se.ToString()));
} catch (Exception e) {
Logger.Write(Logger.Tag.ERROR, string.Format("ArgumentNullException : {0}", e.ToString()));
}
} catch (Exception e) {
Logger.Write(Logger.Tag.ERROR, e.ToString());
} finally {
try
{
// Release the socket.
sender.Shutdown(SocketShutdown.Both);
sender.Close(0);
} catch {}
}
if (!_notified)
{
HandleFailure(request, rh, voting);
}
}
public void ReadCallback(IAsyncResult ar)
{
try
{
String content = String.Empty;
// Retrieve the state object and the handler socket
// from the asynchronous state object.
StateObject state = (StateObject)ar.AsyncState;
Socket handler = state.workSocket;
// Read data from the client socket.
int bytesRead = handler.EndReceive(ar);
if (bytesRead > 0 && state.Retries < 2)
{
// There might be more data, so store the data received so far.
state.sb.Append(Encoding.ASCII.GetString(
state.buffer, 0, bytesRead));
// Check for end-of-file tag. If it is not there, read
// more data.
content = state.sb.ToString();
if (content.IndexOf("<EOF>") > -1 || state.Retries > 0)
{
// All the data has been read from the
// client.
Response response1 = new Response()
{
Completed = null, Data = null, Status = null
};;
try
{
ProcedureCallBatch requestBatch = Encoder.DecodeProcedureCallBatch(content);
if (requestBatch != null)
{
StateLog.Instance.Nodes.AddIfNew(requestBatch.Sender);
response1 = MakeResponse(requestBatch);
}
} catch (Exception e)
{
Console.WriteLine(e.ToString());
}
Send(handler, Encoder.EncodeResponse(response1));
}
else
{
// Not all data received. Get more.
state.Retries += 1;
handler.BeginReceive(state.buffer, 0, StateObject.BufferSize, 0,
new AsyncCallback(ReadCallback), state);
}
}
} catch (Exception e)
{
if (!e.GetType().IsAssignableFrom(typeof(SocketException)))
{
Logger.Write(Logger.Tag.ERROR, e.ToString());
}
}
}
private Response MakeResponse(ProcedureCallBatch requestBatch)
{
Dictionary <string, bool> response = new Dictionary <string, bool>();
if (requestBatch.Sender == null)
{
foreach (Call request in requestBatch.Batch)
{
if (request.Type == Call._Type.VALUE_RESPONSE)
{
ValueResponseBatch vr = Encoder.DecodeValueResponseBatch(request.Data);
if (CurrentState.Instance.IsLeader)
{
StateLog.Instance.Leader_AddActionCompleted(vr.UUID, Params.ID);
}
else
{
StateLog.Instance.Follower_MarkActionCompleted(vr.UUID);
}
StateLog.Instance.RemoveCurrentTask(vr.UUID);
if (Params.TEST_RECEIVER_HOST != string.Empty)
{
try
{
TestReceiverClient.Instance.AddEntry(vr.Responses);
} catch (Exception e)
{
Logger.Write(Logger.Tag.ERROR, "ADD_ENTRY: " + e.ToString());
}
}
response.Add(request.ID, true);
}
else
{
response.Add(request.ID, false);
}
}
return(new Response()
{
Status = response
});
}
else
{
CurrentState.Instance.SetCandidateResolve(false);
HashSet <string> cpa = new HashSet <string>();
if (requestBatch.Completed != null)
{
cpa = new HashSet <string>(requestBatch.Completed);
foreach (string actionID in cpa)
{
StateLog.Instance.Follower_AddActionCompleted(actionID);
}
}
foreach (Call request in requestBatch.Batch)
{
if (request.Type == Call._Type.VOTE)
{
Vote vote = Encoder.DecodeVote(request.Data);
Vote myVote = vote;
int count = StateLog.Instance.LogCount;
if (vote.LogCount < count)
{
myVote = new Vote()
{
ID = Params.ID,
LogCount = count
};
}
else if (Utils.IsCandidate(CurrentState.Instance.Get_State.State))
{
Params.OverwriteParameters(vote.Parameters);
CurrentState.Instance.CancelState();
CurrentState.Instance.Timer.Reset(((int)(Params.HEARTBEAT_MS / 2)));
}
else
{
CurrentState.Instance.Timer.Reset();
}
response.Add(request.ID, true);
Logger.Write(Logger.Tag.WARN, "Received VOTE (OVERWROTE PARAMETERS).");
return(new Response()
{
Status = response,
Data = Encoder.EncodeVote(myVote),
HIEVar = CurrentState.Instance.HIEVar
});
}
else if (request.Type == Call._Type.DATA_REQUEST)
{
DataRequest action = Encoder.DecodeDataRequest(request.Data);
// action.T1 = Utils.Micros.ToString();
bool b = true;
if (!cpa.Contains(action.ID))
{
b = StateLog.Instance.AppendAction(action);
}
CurrentState.Instance.CancelState();
response.Add(request.ID, b);
}
else if (request.Type == Call._Type.NODE_ADD)
{
Credentials node = Encoder.DecodeNode(request.Data);
StateLog.Instance.Nodes.AddNewNode(node);
// if (CurrentState.Instance.IsLeader)
// CurrentState.Instance.CancelState();
response.Add(request.ID, true);
}
else if (request.Type == Call._Type.NODE_DEL)
{
Credentials node = Encoder.DecodeNode(request.Data);
StateLog.Instance.ClearPeerLog(node.ID);
StateLog.Instance.Nodes.TryRemoveNode(node.ID);
CurrentState.Instance.CancelState();
response.Add(request.ID, true);
}
else
{
response.Add(request.ID, false);
}
}
if (requestBatch.Sender != null && requestBatch.Step > Params.STEP)
{
Params.STEP = requestBatch.Step;
}
List <string> completed = null;
if (requestBatch.Sender != null)
{
completed = StateLog.Instance.Follower_GetCompletedActions();
CurrentState.Instance.Timer.Reset();
}
return(new Response()
{
Status = response,
Completed = completed,
HIEVar = CurrentState.Instance.HIEVar
});
}
}
public void OnResponse(string receiverID, ProcedureCallBatch sender, string response)
{
Console.WriteLine(response);
Response r0 = Encoder.DecodeResponse(response);
// Console.WriteLine(request.Type);
}
public static byte[] EncodeProcedureCallBatch(ProcedureCallBatch request)
{
return(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(request, Formatting.None) + "<EOF>"));
}
public void OnResponse(string receiverID, ProcedureCallBatch sender, string response)
{
if (response == null)
{
return;
}
Response r0 = Encoder.DecodeResponse(response);
if (r0 == null || r0.Status == null)
{
// Will never be leader (well, not until next term)
return;
}
sender.Batch.ForEach((r) => {
if (r0.Status.ContainsKey(r.ID))
{
if (r0.Status[r.ID])
{
Vote v0 = Encoder.DecodeVote(r0.Data);
lock (_lock2)
{
votes.Add(v0.ID);
}
}
else
{
lock (_lock2)
{
votes.Add("*");
}
}
}
});
lock (_lock2)
{
Node node;
bool nb = StateLog.Instance.Nodes.TryGetNode(receiverID, out node);
if (nb)
{
node.HIEVar = r0.HIEVar;
}
// foreach(string s in votes) Console.WriteLine(s);
// if (CurrentState.Instance.ReceviedVote)
// {
// CurrentState.Instance.CancelState();
// CurrentState.Instance.Timer.Reset(((int)(Params.HEARTBEAT_MS/2)));
// StateMachine.SetElectionTerm(false);
// }
// if (votes.Count == Quorum && votes.Any((s) => s != Params.ID))
HashSet <string> _votes = new HashSet <string>(votes);
if (votes.Count == Quorum && _votes.Count == 1 && votes.Contains("*") && !CurrentState.Instance.CandidateResolve)
{
CurrentState.Instance.ActAsSleeper();
CurrentState.Instance.SetCandidateResolve(true);
}
else if (votes.Count == Quorum && _votes.Count > 0 && votes.Contains(Params.ID) && Utils.IsCandidate(CurrentState.Instance.Get_State.State))
{
CurrentState.Instance.SetStateAsLeader();
CurrentState.Instance.SetCandidateResolve(false);
}
else if (votes.Count == Quorum && !CurrentState.Instance.CandidateResolve)
{
CurrentState.Instance.CancelState();
CurrentState.Instance.Timer.Reset(((int)(Params.HEARTBEAT_MS / 2)));
}
}
}
private void Loop()
{
bool morePeers = (StateLog.Instance.Nodes.NodeCount > 0);
bool timerActive = CurrentState.Instance.Timer.Active;
(State State, bool TimePassed)state = CurrentState.Instance.Get_State;
if (!morePeers && timerActive)
{
CurrentState.Instance.Timer.End();
_sleepingTimer.Begin();
CurrentState.Instance.ActAsSleeper();
// Logger.Write(Logger.Tag.WARN, state.State.ToString());
}
else if (morePeers && !timerActive)
{
CurrentState.Instance.Timer.Begin();
_sleepingTimer.End();
}
if (Utils.IsCandidate(state.State) || (CurrentState.Instance.CandidateResolve && state.State == State.SLEEPER)) // && !ElectionTerm)
{
// CurrentState.Instance.SetReceivedVote(false);
List <Call> batch = new List <Call> {
new Call()
{
Type = Call._Type.VOTE,
ID = Utils.GenerateUUID(),
Data = Encoder.EncodeVote(new Vote()
{
ID = Params.ID,
LogCount = StateLog.Instance.LogCount
})
}
};
VoteResponseHandler handler = new VoteResponseHandler()
{
Quorum = StateLog.Instance.Nodes.NodeCount
};
Task[] tasks;
StateLog.Instance.Nodes.ForEachAsync((p) => {
ProcedureCallBatch rb = new ProcedureCallBatch()
{
Batch = batch,
Sender = Credentials.Self
};
p.Client.StartClient(rb, handler, true);
}, out tasks);
Parallel.ForEach <Task>(tasks, (t) => { t.Start(); });
Task.WaitAll(tasks);
// ElectionTerm = true;
}
if (CurrentState.Instance.IsLeader || (CurrentState.Instance.CandidateResolve && state.State == State.SLEEPER))
{
bool connectionsLost = false;
StateLog.Instance.Nodes.ForEachPeer((p) => {
if (p.Heartbeat.Value >= Params.MAX_RETRIES)
{
connectionsLost = true;
if (StateLog.Instance.Nodes.NodeCount > 1)
{
StateLog.Instance.AddBehindToAllButOne(p.Client.ID, new Call()
{
Type = Call._Type.NODE_DEL,
ID = Utils.GenerateUUID(),
Data = Encoder.EncodeNode(p.AsNode)
});
}
Node peer = null;
bool success = StateLog.Instance.Nodes.TryGetNode(p.Client.ID, out peer);
Dictionary <string, DataRequest> _rescheduledActions = new Dictionary <string, DataRequest>();
if (success && !CurrentState.Instance.CandidateResolve)
{
peer.ForEachEntry((a) => {
if (!_rescheduledActions.ContainsKey(a.ID))
{
DataRequest action = a;
action.Assigned = StateLog.Instance.Nodes.ScheduleRequest(p.Client.ID);
action.ID = Utils.GenerateUUID();
StateLog.Instance.AddBehindToAllButOne(p.Client.ID, new Call()
{
Type = Call._Type.DATA_REQUEST,
ID = Utils.GenerateUUID(),
Data = Encoder.EncodeDataRequest(action)
});
_rescheduledActions.Add(a.ID, action);
}
});
}
else if (success && CurrentState.Instance.CandidateResolve)
{
peer.ForEachEntry((a) => {
if (!_rescheduledActions.ContainsKey(a.ID))
{
DataRequest action = a;
action.Assigned = Params.ID;
action.ID = Utils.GenerateUUID();
StateLog.Instance.AddBehindToAllButOne(p.Client.ID, new Call()
{
Type = Call._Type.DATA_REQUEST,
ID = Utils.GenerateUUID(),
Data = Encoder.EncodeDataRequest(action)
});
_rescheduledActions.Add(a.ID, action);
}
});
}
StateLog.Instance.ClearPeerLog(p.Client.ID);
StateLog.Instance.Nodes.TryRemoveNode(p.Client.ID);
if (StateLog.Instance.Nodes.NodeCount < 1)
{
CurrentState.Instance.SetCandidateResolve(false);
}
foreach (var action in _rescheduledActions.Values)
{
StateLog.Instance.AppendAction(action);
}
}
else
{
connectionsLost = StateLog.Instance.BatchesBehindCount(p.Client.ID) > 0;
}
});
if (CurrentState.Instance.IsLeader)
{
bool ready = StateLog.Instance.NotAnyBatchOrCompleteBehind();
bool passed = CurrentState.Instance.Timer.HasTimePassed(((int)(Params.HEARTBEAT_MS / 2)));
if (!ready || passed || connectionsLost || StateLog.Instance.Nodes.PeerLogCount > StateLog.Instance.Nodes.NodeCount)
{
Task[] tasks;
StateLog.Instance.Nodes.ForEachAsync((p) => {
HashSet <string> acp = StateLog.Instance.Leader_GetActionsCompleted(p.Client.ID);
ProcedureCallBatch rb = new ProcedureCallBatch()
{
Batch = StateLog.Instance.GetBatchesBehind(p.Client.ID),
Completed = acp,
Sender = Credentials.Self
};
p.Client.StartClient(rb, new DefaultHandler());
}, out tasks);
Parallel.ForEach <Task>(tasks, (t) => { t.Start(); });
Task t = Task.WhenAll(tasks);
try {
t.Wait();
}
catch {}
}
if (passed)
{
Producer.Instance.Broadcast();
CurrentState.Instance.Timer.Reset();
}
if (wait < Utils.Millis)
{
ready = StateLog.Instance.NotAnyBatchOrCompleteBehind();
_consumer.MarkReady(ready);
}
}
}
if (state.State == State.SLEEPER && _sleepingTimer.HasTimePassed(((int)(Params.HEARTBEAT_MS / 2))))
{
try
{
Producer.Instance.Broadcast();
_sleepingTimer.Reset();
} catch {}
}
if (state.State == State.SLEEPER)
{
_consumer.MarkReady(true);
}
if (Params.RUN_HIE)
{
int balance = StateLog.Instance.Nodes.NodeCount * 2;
if (balance < 1)
{
balance = 1;
}
for (int bs = 0; bs < balance; bs++)
{
DataRequest prioritizedAction = StateLog.Instance.PriorityQueue.Dequeue();
if (prioritizedAction != null)
{
List <Driver> requiredDriver = HIE.GetOrCreateDriver(prioritizedAction);
requiredDriver.ForEach((dr0) => { dr0.AddRequestBehind(prioritizedAction); });
}
}
List <Task> tasks = new List <Task>();
HIE.ForEachDriver((d) => {
Task t = new Task(
() => {
d.Write();
}
);
tasks.Add(t);
});
Parallel.ForEach <Task>(tasks, (t) => { t.Start(); });
// Task ta = Task.WhenAll(tasks);
// try {
// ta.Wait();
// }
// catch {}
// List<Task> tasks0 = new List<Task>();
// List<Driver> driver_replacements = new List<Driver>();
// HIE.ForEachDriver((d) => {
// Task t = new Task(
// ()=>{
// if (d.Heartbeat.Value > Params.MAX_RETRIES+1)
// {
// Driver d0 = Driver.MakeDriver(d.Image, d.Config.Replica+1);
// driver_replacements.Add(d0);
// }
// }
// );
// tasks0.Add(t);
// });
// Parallel.ForEach<Task>(tasks0, (t) => { t.Start(); });
// Task ta1 = Task.WhenAll(tasks0);
// try {
// ta.Wait();
// }
// catch {}
TestReceiverClient.Instance.StartClient();
}
// if (FileLogger.Instance.IsEnabled && Utils.Millis > FileLogTS+1000 && !CurrentState.Instance.IsLeader)
// {
// long f = Utils.Millis;
// int i0 = StateLog.Instance.ActionCount;
// int lc = StateLog.Instance.LogCount;
// int i1 = StateLog.Instance.PriorityQueue.Count;
// int i2 = StateLog.Instance.Peers.NodeCount;
// string ram = Utils.MemoryUsage.ToString();
// string cpu = Utils.CPUUsage.ToString();
// FileLogger.Instance.WriteToFile(
// string.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
// f.ToString(), i0.ToString(), lc.ToString(), i1.ToString(), i2.ToString(), state.State.ToString(), cpu, ram)
// );
// FileLogTS = Utils.Millis;
// }
}