public StateLogEntrySuggestion GetNextStateLogEntrySuggestion(StateLogEntryRequest req)
{
StateLogEntrySuggestion le = new StateLogEntrySuggestion()
{
LeaderTerm = this.stateMachine.NodeTerm
};
ulong prevId = 0;
ulong prevTerm = 0;
StateLogEntry entry = null;
if (req.StateLogEntryId == 0)
{
//send first record to sync
var key = GetKey(0, 0);
Slice value;
var iter = db.NewIterator(ReadOptions.Default);
iter.Seek(key);
iter.Next();
while (iter.Valid())
{
entry = StateLogEntry.BiserDecode(value.ToArray());
if (entry.IsCommitted)
{
break;
}
iter.Next();
}
}
else
{
var key = GetKey(req.StateLogEntryTerm, req.StateLogEntryId);
Slice value;
var iter = db.NewIterator(ReadOptions.Default);
iter.Seek(key);
var lastOne = iter.Value();
iter.Next();
while (iter.Valid())
{
entry = StateLogEntry.BiserDecode(value.ToArray());
if (entry.Index > req.StateLogEntryId)
{
var oldEntry = StateLogEntry.BiserDecode(lastOne.ToArray());
prevId = oldEntry.Index;
prevTerm = oldEntry.Term;
}
}
}
if (entry != null)
{
le.StateLogEntry = entry;
entry.PreviousStateLogId = prevId;
entry.PreviousStateLogTerm = prevTerm;
le.IsCommitted = entry.IsCommitted;
return(le);
}
else
{
return(null);
}
}
/// <summary>
/// Is called from lock_operations
/// Adds to silo table, until is moved to log table.
/// This table can be cleared up on start
/// returns concatenated term+index inserted identifier
/// </summary>
/// <param name="data"></param>
/// <param name="externalID">if set up must be returned in OnCommitted to notify that command is executed</param>
/// <returns></returns>
public StateLogEntry AddStateLogEntryForDistribution(byte[] data, byte[] externalID = null)
{
/*
* Only nodes of the current term can be distributed
*/
tempPrevStateLogId = tempStateLogId;
tempPrevStateLogTerm = tempStateLogTerm;
tempStateLogId++;
tempStateLogTerm = rn.NodeTerm;
StateLogEntry le = new StateLogEntry()
{
Index = tempStateLogId,
Data = data,
Term = tempStateLogTerm,
PreviousStateLogId = tempPrevStateLogId,
PreviousStateLogTerm = tempPrevStateLogTerm,
ExternalID = externalID
};
qDistribution.Add(le.Index, le);
return(le);
}
/// <summary>
/// +
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logEntryId"></param>
/// <returns></returns>
public StateLogEntry GetCommitedEntryByIndex(ulong logEntryId)
{
try
{
if (this.LastCommittedIndex < logEntryId)
{
return(null);
}
StateLogEntry isle = null;
lock (inMem.Sync)
{
foreach (var el in inMem.SelectForwardFromTo(logEntryId, ulong.MinValue, true, ulong.MaxValue, ulong.MaxValue))
{
if (el.Item3.FakeEntry)
{
continue;
}
return(el.Item3);
}
return(isle);
}
}
catch (Exception ex)
{
throw ex;
}
}
public bool CommitLogEntry(NodeRaftAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (this.LastCommittedIndex < applied.StateLogEntryId && stateMachine.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
var key = GetKey(applied.StateLogEntryTerm, applied.StateLogEntryId);
var iter = db.NewIterator();
iter.Seek(key);
int update = 0;
while (iter.Valid())
{
var entry = StateLogEntry.BiserDecode(iter.Value());
if (entry.IsCommitted)
{
break;
}
entry.IsCommitted = true;
var value = entry.BiserEncode();
db.Put(key, value);
update++;
iter.Prev();
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
return(update > 0);
}
return(false);
}
public ReturnValueBase ExecuteBusinessLogic(StateLogEntry entry, RaftStateMachine node)
{
node.NodeStateLog.LastBusinessLogicCommittedIndex = entry.Index;
if (GlobalConfig.Verbose)
{
Console.WriteLine("receive entry:" + entry.Index + " on node:" + node.NodeName);
}
string text = System.Text.Encoding.UTF8.GetString(entry.Data);
var cmd = Newtonsoft.Json.JsonConvert.DeserializeObject <LockOper>(text);
if (cmd != null)
{
bool succ = this.table.GetQueue(cmd.Key).LockNoWait(cmd.Session, cmd.Type);
return(new ReturnValueBase
{
Success = succ,
Message = "operation finished"
});
}
return(new ReturnValueBase()
{
Success = false,
Message = "wrong command"
});
}
public StateLogEntry GetEntryByIndexTerm(ulong logEntryId, ulong logEntryTerm)
{
var key = GetKey(logEntryTerm, logEntryId);
Slice value;
var find = db.TryGet(ReadOptions.Default, (Slice)key, out value);
if (find)
{
var entry = StateLogEntry.BiserDecode(value.ToArray());
return(entry);
}
return(null);
}
public StateLogEntry GetEntryByIndexTerm(ulong logEntryId, ulong logEntryTerm)
{
var key = GetKey(logEntryTerm, logEntryId);
var value = db.Get(key);
var find = value == null;
if (find)
{
var entry = StateLogEntry.BiserDecode(value);
return(entry);
}
return(null);
}
public SyncResult SyncCommitByHeartBeat(LeaderHeartbeat lhb)
{
if (GlobalConfig.Verbose)
{
Console.WriteLine($"leader info:{lhb.LastStateLogCommittedIndex} ,mine:{this.LastCommittedIndex}");
}
if (this.LastCommittedIndex < lhb.LastStateLogCommittedIndex)
{
//find if this entry exist
var key = GetKey(lhb.LastStateLogCommittedIndexTerm, lhb.StateLogLatestIndex);
var value = db.Get(key);
var find = value == null;
if (find)
{
var entry = StateLogEntry.BiserDecode(value);
if (!entry.IsCommitted)
{
entry.IsCommitted = true;
value = entry.BiserEncode();
db.Put(key, value);
this.LastCommittedIndex = lhb.LastStateLogCommittedIndex;
return(new SyncResult()
{
HasCommit = true, Synced = true
});
}
else
{
return(new SyncResult()
{
HasCommit = false, Synced = true
});
}
}
else
{
return(new SyncResult()
{
HasCommit = false, Synced = false
});
}
}
else
{
return(new SyncResult()
{
HasCommit = false, Synced = true
});
}
}
/// <summary>
/// +
/// Get Term by EntryLogIndex. Returns First element false if not found, Second - Term (if found).
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logId"></param>
/// <returns>first element true if exists</returns>
public StateLogEntry GetEntryByIndexTerm(ulong logEntryId, ulong logEntryTerm)
{
try
{
Tuple <ulong, StateLogEntry> sleTpl;
StateLogEntry isle = null;
lock (inMem.Sync)
{
inMem.Select(logEntryId, logEntryTerm, out isle);
}
return(isle);
}
catch (Exception ex)
{
throw ex;
}
}
public StateLogEntrySuggestion GetNextStateLogEntrySuggestion(StateLogEntryRequest req)
{
StateLogEntrySuggestion le = new StateLogEntrySuggestion()
{
LeaderTerm = this.stateMachine.NodeTerm
};
ulong prevId = 0;
ulong prevTerm = 0;
StateLogEntry entry = null;
if (req.StateLogEntryId == 0)
{
//send first record to sync
entry = list.Find(s => s.IsCommitted == true);
}
else
{
for (int index = 0; index < list.Count; index++)
{
if (list[index].Index > req.StateLogEntryId)
{
//find next one and set preview value
if (index > 0)
{
prevId = list[index - 1].Index;
prevTerm = list[index - 1].Term;
entry = list[index];
break;
}
}
}
}
if (entry != null)
{
le.StateLogEntry = entry;
entry.PreviousStateLogId = prevId;
entry.PreviousStateLogTerm = prevTerm;
le.IsCommitted = entry.IsCommitted;
return(le);
}
else
{
return(null);
}
}
public void RollbackToLastestCommit()
{
if (LastCommittedIndex == 0 || LastCommittedIndexTerm == 0)
{
return;
}
var key = GetKey(0, 0);
Slice value;
var iter = db.NewIterator(ReadOptions.Default);
iter.SeekToLast();
while (iter.Valid())
{
var entry = StateLogEntry.BiserDecode(iter.Value().ToArray());
if (entry.Term >= LastCommittedIndexTerm && entry.Index > LastCommittedIndex)
{
db.Delete(WriteOptions.Default, iter.Key());
}
}
}
public void RollbackToLastestCommit()
{
if (LastCommittedIndex == 0 || LastCommittedIndexTerm == 0)
{
return;
}
var key = GetKey(0, 0);
var iter = db.NewIterator();
iter.SeekToLast();
while (iter.Valid())
{
var entry = StateLogEntry.BiserDecode(iter.Value());
if (entry.Term >= LastCommittedIndexTerm && entry.Index > LastCommittedIndex)
{
db.Remove(iter.Key());
}
}
}
public void AddLogEntry(StateLogEntrySuggestion suggestion)
{
PreviousStateLogId = suggestion.StateLogEntry.PreviousStateLogId;
PreviousStateLogTerm = suggestion.StateLogEntry.PreviousStateLogTerm;
StateLogId = suggestion.StateLogEntry.Index;
StateLogTerm = suggestion.StateLogEntry.Term;
var key = GetKey(suggestion.StateLogEntry.Term, suggestion.StateLogEntry.Index);
var value = db.Get(key);
var find = value != null;
if (find)
{
var entry = StateLogEntry.BiserDecode(value);
entry.IsCommitted = suggestion.IsCommitted;
value = entry.BiserEncoder().Encode();
db.Put(key, value);
}
else
{
var data = suggestion.StateLogEntry.BiserEncode();
db.Put(key, data);
}
}
public void ReloadFromStorage()
{
//should load from log files
var iter = db.NewIterator();
iter.SeekToLast();
while (iter.Valid())
{
var entry = StateLogEntry.BiserDecode(iter.Value());
StateLogId = entry.Index;
StateLogTerm = entry.Term;
PreviousStateLogId = entry.PreviousStateLogId;
PreviousStateLogTerm = entry.PreviousStateLogTerm;
stateMachine.NodeTerm = entry.Term;
if (entry.IsCommitted)
{
LastCommittedIndex = entry.Index;
LastCommittedIndexTerm = entry.Term;
LastBusinessLogicCommittedIndex = entry.Index;
return;
}
}
}
public void AddLogEntry(StateLogEntrySuggestion suggestion)
{
PreviousStateLogId = suggestion.StateLogEntry.PreviousStateLogId;
PreviousStateLogTerm = suggestion.StateLogEntry.PreviousStateLogTerm;
StateLogId = suggestion.StateLogEntry.Index;
StateLogTerm = suggestion.StateLogEntry.Term;
var key = GetKey(suggestion.StateLogEntry.Term, suggestion.StateLogEntry.Index);
Slice value;
var find = db.TryGet(ReadOptions.Default, (Slice)key, out value);
if (find)
{
var entry = StateLogEntry.BiserDecode(value.ToArray());
entry.IsCommitted = suggestion.IsCommitted;
value = (Slice)entry.BiserEncoder().Encode();
db.Put(WriteOptions.Default, key, value);
}
else
{
var data = suggestion.StateLogEntry.BiserEncode();
db.Put(WriteOptions.Default, key, (Slice)data);
}
}
static void t1()
{
/*
* Start
* Protobuf obj length: 22
* Biser Binary obj length: 17
* NetJson obj length: 129
* Biser Json obj length: 129
*
* Protobuf encode: 1184 ms
* Protobuf decode: 1569 ms
* Biser Binary encode: 396 ms
* Biser Binary decode: 209 ms
* NetJson encode: 1350 ms
* NetJson decode: 1902 ms
* Biser Json encode: 2266 ms
* Biser Json decode: 3659 ms
* Press any key
*/
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
// It's an operational class from https://github.com/hhblaze/Raft.Net/blob/master/Raft/StateMachine/StateLogEntry.cs
StateLogEntry obj = new StateLogEntry()
{
Data = new byte[] { 1, 2, 3, 4, 5 },
Index = 458,
IsCommitted = true,
PreviousStateLogId = 4789,
PreviousStateLogTerm = 447,
RedirectId = 12,
Term = 99
};
Console.WriteLine("t1----------------------------------");
//Protobuf. Warming up, getting length
var pBt = obj.SerializeProtobuf();
Console.WriteLine($"Protobuf obj length: {pBt.Length}");
var pObj = pBt.DeserializeProtobuf <StateLogEntry>();
//Biser. Getting length
var bBt = new Biser.Encoder().Add(obj).Encode();
Console.WriteLine($"Biser Binary obj length: {bBt.Length}");
var bObj = StateLogEntry.BiserDecode(bBt);
//NetJson. Getting length
var njss = NetJSON.NetJSON.Serialize(obj);
Console.WriteLine($"NetJson obj length: {System.Text.Encoding.UTF8.GetBytes(njss).Length}");
var bnjss = NetJSON.NetJSON.Deserialize <StateLogEntry>(njss);
//Biser Json. Getting length
var bjss = new Biser.JsonEncoder(obj).GetJSON();
Console.WriteLine($"Biser Json obj length: {System.Text.Encoding.UTF8.GetBytes(bjss).Length}");
var bbjss = StateLogEntry.BiserJsonDecode(bjss);
//Message Pack
var mBt = MessagePackSerializer.Serialize(obj);
Console.WriteLine($"Message Pack obj length: {mBt.Length}");
var mc2 = MessagePackSerializer.Deserialize <StateLogEntry>(mBt);
Console.WriteLine("");
byte[] tbt = null;
StateLogEntry tobj = null;
sw.Start();
for (int i = 0; i < 1000000; i++)
{
tbt = obj.SerializeProtobuf();
}
sw.Stop();
Console.WriteLine($"Protobuf encode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
tobj = pBt.DeserializeProtobuf <StateLogEntry>();
}
sw.Stop();
Console.WriteLine($"Protobuf decode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
tbt = new Biser.Encoder().Add(obj).Encode();
}
sw.Stop();
Console.WriteLine($"Biser Binary encode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
tobj = StateLogEntry.BiserDecode(bBt);
}
sw.Stop();
Console.WriteLine($"Biser Binary decode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
njss = NetJSON.NetJSON.Serialize(obj);
}
sw.Stop();
Console.WriteLine($"NetJson encode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
bnjss = NetJSON.NetJSON.Deserialize <StateLogEntry>(njss);
}
sw.Stop();
Console.WriteLine($"NetJson decode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
bjss = new Biser.JsonEncoder(obj).GetJSON();
}
sw.Stop();
Console.WriteLine($"Biser Json encode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
bbjss = StateLogEntry.BiserJsonDecode(bjss);
}
sw.Stop();
Console.WriteLine($"Biser Json decode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
mBt = MessagePackSerializer.Serialize(obj);
}
sw.Stop();
Console.WriteLine($"MessagePack encode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
for (int i = 0; i < 1000000; i++)
{
mc2 = MessagePackSerializer.Deserialize <StateLogEntry>(mBt);
}
sw.Stop();
Console.WriteLine($"MessagePack decode: {sw.ElapsedMilliseconds} ms");
sw.Reset();
}
internal void Commited()
{
if (System.Threading.Interlocked.CompareExchange(ref this.stateMachine.States.inCommit, 1, 0) != 0)
{
return;
}
Task.Run(async() =>
{
StateLogEntry sle = null;
while (true)
{
if (GlobalConfig.Verbose)
{
Console.WriteLine("in commited");
}
lock (this.stateMachine.lock_Operations)
{
if (this.stateMachine.NodeStateLog.LastCommittedIndex == this.stateMachine.NodeStateLog.LastBusinessLogicCommittedIndex)
{
System.Threading.Interlocked.Exchange(ref this.stateMachine.States.inCommit, 0);
return;
}
else
{
sle = this.stateMachine.NodeStateLog.GetCommitedEntryByIndex(
this.stateMachine.NodeStateLog.LastCommittedIndexTerm,
this.stateMachine.NodeStateLog.LastBusinessLogicCommittedIndex + 1);
if (sle == null)
{
System.Threading.Interlocked.Exchange(ref this.stateMachine.States.inCommit, 0);
return;
}
}
}
try
{
if (GlobalConfig.Verbose)
{
Console.WriteLine("before call business logic");
}
var executeResult = this.businessLogicHandler.ExecuteBusinessLogic(sle, this.stateMachine);
//In case if business logic commit was successful
//Notifying Async AddLog
//if (GlobalConfig.Verbose)
//{
// Console.WriteLine("before release lock");
//}
if (sle.ExternalID != null && AsyncResponseHandler.df.TryGetValue(sle.ExternalID.ToBytesString(), out var responseCrate))
{
responseCrate.IsRespOk = true;
responseCrate.ReturnValue = executeResult;
responseCrate.res = sle.ExternalID;
responseCrate.Set_MRE();
}
//else
//{
// await Task.Delay(500);
// //repeating with the same id
//}
}
catch (Exception ex)
{
this.stateMachine.Log.Log(new WarningLogEntry()
{
Exception = ex, Method = "Raft.RaftNode.Commited"
});
//Notifying Async AddLog
if (sle.ExternalID != null && AsyncResponseHandler.df.TryGetValue(sle.ExternalID.ToBytesString(), out var responseCrate))
{
responseCrate.IsRespOk = false;
responseCrate.res = sle.ExternalID;
responseCrate.Set_MRE();
}
}
}
});
}
