public static StateLogEntryApplied BiserDecode(byte[] enc = null, Biser.Decoder extDecoder = null) //!!!!!!!!!!!!!! change return type
{
Biser.Decoder decoder = null;
if (extDecoder == null)
{
if (enc == null || enc.Length == 0)
{
return(null);
}
decoder = new Biser.Decoder(enc);
if (decoder.CheckNull())
{
return(null);
}
}
else
{
if (extDecoder.CheckNull())
{
return(null);
}
else
{
decoder = extDecoder;
}
}
StateLogEntryApplied m = new StateLogEntryApplied(); //!!!!!!!!!!!!!! change return type
m.StateLogEntryId = decoder.GetULong();
m.StateLogEntryTerm = decoder.GetULong();
return(m);
}
/// <summary>
/// Only for Follower
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntrySuggestion(NodeAddress address, byte[] data)
{
if (this.NodeState != eNodeState.Follower)
{
return;
}
StateLogEntrySuggestion suggest = StateLogEntrySuggestion.BiserDecode(data); //data.DeserializeProtobuf<StateLogEntrySuggestion>();
if (this.NodeTerm > suggest.LeaderTerm) //Sending Leader is not Leader anymore
{
this.NodeStateLog.LeaderSynchronizationIsActive = false;
return;
}
if (this.NodeTerm < suggest.LeaderTerm)
{
this.NodeTerm = suggest.LeaderTerm;
}
if (suggest.StateLogEntry.Index <= NodeStateLog.LastCommittedIndex) //Preventing same entry income, can happen if restoration was sent twice (while switch of leaders)
{
return; //breakpoint don't remove
}
//Checking if node can accept current suggestion
if (suggest.StateLogEntry.PreviousStateLogId > 0)
{
var sle = this.NodeStateLog.GetEntryByIndexTerm(suggest.StateLogEntry.PreviousStateLogId, suggest.StateLogEntry.PreviousStateLogTerm);
if (sle == null)
{
//We don't have previous to this log and need new index request
//VerbosePrint($"{NodeAddress.NodeAddressId}> in sync 1 ");
if (entitySettings.InMemoryEntity && entitySettings.InMemoryEntityStartSyncFromLatestEntity && this.NodeStateLog.LastAppliedIndex == 0)
{
//helps newly starting mode with specific InMemory parameters get only latest command for the entity
}
else
{
this.SyncronizeWithLeader();
return;
}
}
}
//We can apply new Log Entry from the Leader and answer successfully
this.NodeStateLog.AddToLogFollower(suggest);
StateLogEntryApplied applied = new StateLogEntryApplied()
{
StateLogEntryId = suggest.StateLogEntry.Index,
StateLogEntryTerm = suggest.StateLogEntry.Term
// RedirectId = suggest.StateLogEntry.RedirectId
};
//this.NodeStateLog.LeaderSynchronizationIsActive = false;
this.Sender.SendTo(address, eRaftSignalType.StateLogEntryAccepted, applied.SerializeBiser(), this.NodeAddress, entitySettings.EntityName);
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public bool CommitLogEntry(NodeRaftAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (this.LastCommittedIndex < applied.StateLogEntryId && statemachine.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
using (var t = this.db.GetTransaction())
{
//Gathering all not commited entries that are bigger than latest committed index
t.ValuesLazyLoadingIsOn = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
{
lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
}
t.Insert <byte[], byte[]>(stateTableName, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
t.Commit();
//qDistribution.Remove(applied.StateLogEntryId);
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
return(lstCommited.Count > 0);
}
return(false);
}
/// <summary>
/// Only for Follower
/// Is called from tryCatch and in lock
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntrySuggestion(NodeRaftAddress address, object data)
{
if (this.States.NodeState != eNodeState.Follower)
{
return;
}
StateLogEntrySuggestion suggest = data as StateLogEntrySuggestion;
if (this.NodeTerm > suggest.LeaderTerm) //Sending Leader is not Leader anymore
{
this.States.LeaderSynchronizationIsActive = false;
return;
}
if (this.NodeTerm < suggest.LeaderTerm)
{
this.NodeTerm = suggest.LeaderTerm;
}
if (suggest.StateLogEntry == null || (suggest.StateLogEntry.Index <= NodeStateLog.LastCommittedIndex)) //Preventing same entry income, can happen if restoration was sent twice (while switch of leaders)
{
return; //breakpoint don't remove
}
//Checking if node can accept current suggestion
if (suggest.StateLogEntry.PreviousStateLogId > 0)
{
var sle = this.NodeStateLog.GetEntryByIndexTerm(suggest.StateLogEntry.PreviousStateLogId, suggest.StateLogEntry.PreviousStateLogTerm);
if (sle == null)
{
this.SyncronizeWithLeader();
return;
}
}
//We can apply new Log Entry from the Leader and answer successfully
this.logHandler.AddLogEntryByFollower(suggest);
StateLogEntryApplied applied = new StateLogEntryApplied()
{
StateLogEntryId = suggest.StateLogEntry.Index,
StateLogEntryTerm = suggest.StateLogEntry.Term
// RedirectId = suggest.StateLogEntry.RedirectId
};
this.network.SendTo(address, eRaftSignalType.StateLogEntryAccepted, applied, this.NodeAddress, entitySettings.EntityName);
}
/// <summary>
/// Leader receives accepted Log
/// </summary>
/// <param name="address"></param>
/// <param name="data"></param>
void ParseStateLogEntryAccepted(NodeAddress address, byte[] data)
{
if (this.NodeState != eNodeState.Leader)
{
return;
}
StateLogEntryApplied applied = StateLogEntryApplied.BiserDecode(data);
var res = this.NodeStateLog.EntryIsAccepted(address, GetMajorityQuantity(), applied);
if (res == StateLog.eEntryAcceptanceResult.Committed)
{
this.VerbosePrint($"{this.NodeAddress.NodeAddressId}> LogEntry {applied.StateLogEntryId} is COMMITTED (answer from {address.NodeAddressId})");
RemoveLeaderLogResendTimer();
//Force heartbeat, to make followers to get faster info about commited elements
LeaderHeartbeat heartBeat = new LeaderHeartbeat()
{
LeaderTerm = this.NodeTerm,
StateLogLatestIndex = NodeStateLog.StateLogId,
StateLogLatestTerm = NodeStateLog.StateLogTerm,
LastStateLogCommittedIndex = this.NodeStateLog.LastCommittedIndex,
LastStateLogCommittedIndexTerm = this.NodeStateLog.LastCommittedIndexTerm
};
this.Sender.SendToAll(eRaftSignalType.LeaderHearthbeat, heartBeat.SerializeBiser(), this.NodeAddress, entitySettings.EntityName, true);
//---------------------------------------
//this.NodeStateLog.RemoveEntryFromDistribution(applied.StateLogEntryId, applied.StateLogEntryTerm);
InLogEntrySend = false;
ApplyLogEntry();
}
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public eEntryAcceptanceResult EntryIsAccepted(NodeAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (applied.StateLogEntryId <= this.LastCommittedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
if (applied.StateLogEntryId <= this.LastAppliedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
StateLogEntryAcceptance acc = null;
if (dStateLogEntryAcceptance.TryGetValue(applied.StateLogEntryId, out acc))
{
if (acc.Term != applied.StateLogEntryTerm)
{
return(eEntryAcceptanceResult.NotAccepted); //Came from wrong Leader probably
}
//acc.Quantity += 1;
acc.acceptedEndPoints.Add(address.EndPointSID);
}
else
{
acc = new StateLogEntryAcceptance()
{
//Quantity = 2, //Leader + first incoming
Index = applied.StateLogEntryId,
Term = applied.StateLogEntryTerm
};
acc.acceptedEndPoints.Add(address.EndPointSID);
dStateLogEntryAcceptance[applied.StateLogEntryId] = acc;
}
if ((acc.acceptedEndPoints.Count + 1) >= majorityQuantity)
{
this.LastAppliedIndex = applied.StateLogEntryId;
//Removing from Dictionary
dStateLogEntryAcceptance.Remove(applied.StateLogEntryId);
if (this.LastCommittedIndex < applied.StateLogEntryId && rn.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
using (var t = this.rn.db.GetTransaction())
{
//Gathering all not commited entries that are bigger than latest committed index
t.ValuesLazyLoadingIsOn = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
{
lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
}
//Setting latest commited index
if (rn.entitySettings.DelayedPersistenceIsActive)
{
foreach (var iel in sleCache.Where(r => r.Key >= this.LastCommittedIndex + 1 && r.Value.Item1 == applied.StateLogEntryTerm))
{
lstCommited.Add(iel.Value.Item2.Data);
}
sleCacheIndex = applied.StateLogEntryId;
sleCacheTerm = applied.StateLogEntryTerm;
}
else
{
t.Insert <byte[], byte[]>(tblStateLogEntry, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
t.Commit();
}
//Removing entry from command queue
//t.RemoveKey<byte[]>(tblAppendLogEntry, new byte[] { 1 }.ToBytes(applied.StateLogEntryTerm, applied.StateLogEntryId));
qDistribution.Remove(applied.StateLogEntryId);
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
if (lstCommited.Count > 0)
{
this.rn.Commited();
}
//this.rn.Commited(applied.StateLogEntryId);
return(eEntryAcceptanceResult.Committed);
}
}
return(eEntryAcceptanceResult.Accepted);
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public eEntryAcceptanceResult EntryIsAccepted(NodeRaftAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (applied.StateLogEntryId <= this.LastCommittedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
if (applied.StateLogEntryId <= this.LastAppliedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
StateLogEntryAcceptance acc = null;
if (dStateLogEntryAcceptance.TryGetValue(applied.StateLogEntryId, out acc))
{
if (acc.Term != applied.StateLogEntryTerm)
{
return(eEntryAcceptanceResult.NotAccepted); //Came from wrong Leader probably
}
acc.acceptedEndPoints.Add(address.EndPointSID);
}
else
{
acc = new StateLogEntryAcceptance()
{
Index = applied.StateLogEntryId,
Term = applied.StateLogEntryTerm
};
acc.acceptedEndPoints.Add(address.EndPointSID);
dStateLogEntryAcceptance[applied.StateLogEntryId] = acc;
}
if ((acc.acceptedEndPoints.Count + 1) >= majorityQuantity)
{
this.LastAppliedIndex = applied.StateLogEntryId;
//Removing from Dictionary
dStateLogEntryAcceptance.Remove(applied.StateLogEntryId);
if (this.LastCommittedIndex < applied.StateLogEntryId && statemachine.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
var col = this.db.GetCollection <StateLogEntry>(stateTableName);
var list = col.Query().Where(s => s.Index >= this.LastCommittedIndex + 1 && s.Term == applied.StateLogEntryTerm).ToList();
foreach (var item in list)
{
lstCommited.Add(new byte[1]);
item.IsCommitted = true;
col.Update(item);
qDistribution.Remove(applied.StateLogEntryId);
}
//using (var t = this.db.GetTransaction())
//{
// //Gathering all not commited entries that are bigger than latest committed index
// t.ValuesLazyLoadingIsOn = false;
// foreach (var el in t.SelectForwardFromTo<byte[], byte[]>(stateTableName,
// new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
// new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
// {
// lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
// }
// t.Insert<byte[], byte[]>(stateTableName, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
// t.Commit();
// qDistribution.Remove(applied.StateLogEntryId);
//}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
if (lstCommited.Count > 0)
{
this.statemachine.logHandler.Commited();
}
return(eEntryAcceptanceResult.Committed);
}
}
return(eEntryAcceptanceResult.Accepted);
}