public static StateLogEntrySuggestion BiserDecode(byte[] enc = null, Biser.Decoder extDecoder = null) //!!!!!!!!!!!!!! change return type
{
Biser.Decoder decoder = null;
if (extDecoder == null)
{
if (enc == null || enc.Length == 0)
{
return(null);
}
decoder = new Biser.Decoder(enc);
if (decoder.CheckNull())
{
return(null);
}
}
else
{
if (extDecoder.CheckNull())
{
return(null);
}
else
{
decoder = extDecoder;
}
}
StateLogEntrySuggestion m = new StateLogEntrySuggestion(); //!!!!!!!!!!!!!! change return type
m.LeaderTerm = decoder.GetULong();
m.StateLogEntry = StateLogEntry.BiserDecode(extDecoder: decoder);
m.IsCommitted = decoder.GetBool();
return(m);
}
/// <summary>
/// +
/// Get Term by EntryLogIndex. Returns First element false if not found, Second - Term (if found).
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logId"></param>
/// <returns>first element true if exists</returns>
public StateLogEntry GetEntryByIndexTerm(ulong logEntryId, ulong logEntryTerm)
{
try
{
Tuple <ulong, StateLogEntry> sleTpl;
if (
rn.entitySettings.DelayedPersistenceIsActive
&&
sleCache.TryGetValue(logEntryId, out sleTpl) && sleTpl.Item1 == logEntryTerm
)
{
return(sleTpl.Item2);
}
using (var t = this.rn.db.GetTransaction())
{
var row = t.Select <byte[], byte[]>(tblStateLogEntry, new byte[] { 1 }.ToBytes(logEntryId, logEntryTerm));
if (!row.Exists)
{
//foreach (var el in t.SelectForwardStartsWith<byte[], byte[]>(tblStateLogEntry, new byte[] { 1 }))
//{
// Console.WriteLine($"{rn.NodeAddress.NodeAddressId}> GetEntryByIndexTerm-NULL: {el.Key.ToBytesString()}");
//}
return(null);
}
return(StateLogEntry.BiserDecode(row.Value));
}
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public bool CommitLogEntry(NodeRaftAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (this.LastCommittedIndex < applied.StateLogEntryId && statemachine.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
using (var t = this.db.GetTransaction())
{
//Gathering all not commited entries that are bigger than latest committed index
t.ValuesLazyLoadingIsOn = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
{
lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
}
t.Insert <byte[], byte[]>(stateTableName, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
t.Commit();
//qDistribution.Remove(applied.StateLogEntryId);
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
return(lstCommited.Count > 0);
}
return(false);
}
/// <summary>
/// +
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logEntryId"></param>
/// <returns></returns>
public StateLogEntry GetCommitedEntryByIndex(ulong logEntryId)
{
try
{
if (this.LastCommittedIndex < logEntryId)
{
return(null);
}
using (var t = this.db.GetTransaction())
{
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(logEntryId, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true, true))
{
return(StateLogEntry.BiserDecode(el.Value));
}
}
return(null);
}
catch (Exception ex)
{
throw ex;
}
}
public StateLog(RaftNode rn)
{
this.rn = rn;
//if (rn.nodeSettings.InMemoryEntity)
// if (String.IsNullOrEmpty(dbreezePath) || rn.nodeSettings.InMemoryEntity)
// db = new DBreezeEngine(new DBreezeConfiguration { Storage = DBreezeConfiguration.eStorage.MEMORY });
//else
// db = new DBreezeEngine(dbreezePath);
if (rn.entitySettings.EntityName != "default")
{
tblStateLogEntry += "_" + rn.entitySettings.EntityName;
}
if (rn.entitySettings.InMemoryEntity)
{
tblStateLogEntry = "mem_" + tblStateLogEntry;
}
using (var t = this.rn.db.GetTransaction())
{
var row = t.SelectBackwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true,
new byte[] { 1 }.ToBytes(ulong.MinValue, ulong.MinValue), true)
.FirstOrDefault();
StateLogEntry sle = null;
if (row != null && row.Exists)
{
sle = StateLogEntry.BiserDecode(row.Value);
StateLogId = sle.Index;
StateLogTerm = sle.Term;
PreviousStateLogId = sle.PreviousStateLogId;
PreviousStateLogTerm = sle.PreviousStateLogTerm;
tempPrevStateLogId = PreviousStateLogId;
tempPrevStateLogTerm = PreviousStateLogTerm;
tempStateLogId = StateLogId;
tempStateLogTerm = StateLogTerm;
rn.NodeTerm = sle.Term;
}
var rowTerm = t.Select <byte[], byte[]>(tblStateLogEntry, new byte[] { 2 });
if (rowTerm.Exists)
{
LastCommittedIndex = rowTerm.Value.Substring(0, 8).To_UInt64_BigEndian();
LastCommittedIndexTerm = rowTerm.Value.Substring(8, 8).To_UInt64_BigEndian();
}
var rowBL = t.Select <byte[], ulong>(tblStateLogEntry, new byte[] { 3 });
if (rowBL.Exists)
{
LastBusinessLogicCommittedIndex = rowBL.Value;
}
}
}
/// <summary>
/// +
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logEntryId"></param>
/// <returns></returns>
public StateLogEntry GetCommitedEntryByIndex(ulong logEntryId)
{
try
{
if (this.LastCommittedIndex < logEntryId)
{
return(null);
}
Tuple <ulong, StateLogEntry> sleTpl;
if (
rn.entitySettings.DelayedPersistenceIsActive
&&
sleCache.TryGetValue(logEntryId, out sleTpl)
)
{
return(sleTpl.Item2);
}
if (rn.entitySettings.InMemoryEntity)
{
StateLogEntry isle = null;
lock (inMem.Sync)
{
foreach (var el in inMem.SelectForwardFromTo(logEntryId, ulong.MinValue, true, ulong.MaxValue, ulong.MaxValue))
{
if (el.Item3.FakeEntry)
{
continue;
}
return(el.Item3);
}
}
return(isle);
}
using (var t = this.rn.db.GetTransaction())
{
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(logEntryId, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true, true))
{
return(StateLogEntry.BiserDecode(el.Value));
}
}
return(null);
}
catch (Exception ex)
{
throw ex;
}
}
public StateLog(RaftStateMachine rn, DBreezeEngine dbEngine)
{
this.statemachine = rn;
this.db = dbEngine;
if (rn.entitySettings.EntityName != "default")
{
stateTableName += "_" + rn.entitySettings.EntityName;
}
using (var t = this.db.GetTransaction())
{
var row = t.SelectBackwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true,
new byte[] { 1 }.ToBytes(ulong.MinValue, ulong.MinValue), true)
.FirstOrDefault();
StateLogEntry sle = null;
if (row != null && row.Exists)
{
sle = StateLogEntry.BiserDecode(row.Value);
StateLogId = sle.Index;
StateLogTerm = sle.Term;
PreviousStateLogId = sle.PreviousStateLogId;
PreviousStateLogTerm = sle.PreviousStateLogTerm;
//tempPrevStateLogId = PreviousStateLogId;
//tempPrevStateLogTerm = PreviousStateLogTerm;
//tempStateLogId = StateLogId;
//tempStateLogTerm = StateLogTerm;
rn.NodeTerm = sle.Term;
}
var rowTerm = t.Select <byte[], byte[]>(stateTableName, new byte[] { 2 });
if (rowTerm.Exists)
{
LastCommittedIndex = rowTerm.Value.Substring(0, 8).To_UInt64_BigEndian();
LastCommittedIndexTerm = rowTerm.Value.Substring(8, 8).To_UInt64_BigEndian();
}
var rowBL = t.Select <byte[], ulong>(stateTableName, new byte[] { 3 });
if (rowBL.Exists)
{
LastBusinessLogicCommittedIndex = rowBL.Value;
}
}
}
/// <summary>
/// +
/// Get Term by EntryLogIndex. Returns First element false if not found, Second - Term (if found).
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logId"></param>
/// <returns>first element true if exists</returns>
public StateLogEntry GetEntryByIndexTerm(ulong logEntryId, ulong logEntryTerm)
{
try
{
Tuple <ulong, StateLogEntry> sleTpl;
using (var t = this.db.GetTransaction())
{
var row = t.Select <byte[], byte[]>(stateTableName, new byte[] { 1 }.ToBytes(logEntryId, logEntryTerm));
if (!row.Exists)
{
return(null);
}
return(StateLogEntry.BiserDecode(row.Value));
}
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// +
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logEntryId"></param>
/// <returns></returns>
public StateLogEntry GetCommitedEntryByIndex(ulong logEntryId)
{
try
{
if (this.LastCommittedIndex < logEntryId)
{
return(null);
}
Tuple <ulong, StateLogEntry> sleTpl;
if (
rn.nodeSettings.DelayedPersistenceIsActive
&&
sleCache.TryGetValue(logEntryId, out sleTpl)
)
{
return(sleTpl.Item2);
}
using (var t = db.GetTransaction())
{
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(logEntryId, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true, true))
{
return(StateLogEntry.BiserDecode(el.Value));
}
}
return(null);
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// +
/// Only Leader's proc.
/// Accepts entry return true if Committed
/// </summary>
/// <param name="majorityNumber"></param>
/// <param name="LogId"></param>
/// <param name="TermId"></param>
public eEntryAcceptanceResult EntryIsAccepted(NodeAddress address, uint majorityQuantity, StateLogEntryApplied applied)
{
//If we receive acceptance signals of already Committed entries, we just ignore them
if (applied.StateLogEntryId <= this.LastCommittedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
if (applied.StateLogEntryId <= this.LastAppliedIndex)
{
return(eEntryAcceptanceResult.AlreadyAccepted); //already accepted
}
StateLogEntryAcceptance acc = null;
if (dStateLogEntryAcceptance.TryGetValue(applied.StateLogEntryId, out acc))
{
if (acc.Term != applied.StateLogEntryTerm)
{
return(eEntryAcceptanceResult.NotAccepted); //Came from wrong Leader probably
}
//acc.Quantity += 1;
acc.acceptedEndPoints.Add(address.EndPointSID);
}
else
{
acc = new StateLogEntryAcceptance()
{
//Quantity = 2, //Leader + first incoming
Index = applied.StateLogEntryId,
Term = applied.StateLogEntryTerm
};
acc.acceptedEndPoints.Add(address.EndPointSID);
dStateLogEntryAcceptance[applied.StateLogEntryId] = acc;
}
if ((acc.acceptedEndPoints.Count + 1) >= majorityQuantity)
{
this.LastAppliedIndex = applied.StateLogEntryId;
//Removing from Dictionary
dStateLogEntryAcceptance.Remove(applied.StateLogEntryId);
if (this.LastCommittedIndex < applied.StateLogEntryId && rn.NodeTerm == applied.StateLogEntryTerm) //Setting LastCommittedId
{
//Saving committed entry (all previous are automatically committed)
List <byte[]> lstCommited = new List <byte[]>();
using (var t = this.rn.db.GetTransaction())
{
//Gathering all not commited entries that are bigger than latest committed index
t.ValuesLazyLoadingIsOn = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(this.LastCommittedIndex + 1, applied.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, applied.StateLogEntryTerm), true, true))
{
lstCommited.Add(StateLogEntry.BiserDecode(el.Value).Data);
}
//Setting latest commited index
if (rn.entitySettings.DelayedPersistenceIsActive)
{
foreach (var iel in sleCache.Where(r => r.Key >= this.LastCommittedIndex + 1 && r.Value.Item1 == applied.StateLogEntryTerm))
{
lstCommited.Add(iel.Value.Item2.Data);
}
sleCacheIndex = applied.StateLogEntryId;
sleCacheTerm = applied.StateLogEntryTerm;
}
else
{
t.Insert <byte[], byte[]>(tblStateLogEntry, new byte[] { 2 }, applied.StateLogEntryId.ToBytes(applied.StateLogEntryTerm));
t.Commit();
}
//Removing entry from command queue
//t.RemoveKey<byte[]>(tblAppendLogEntry, new byte[] { 1 }.ToBytes(applied.StateLogEntryTerm, applied.StateLogEntryId));
qDistribution.Remove(applied.StateLogEntryId);
}
this.LastCommittedIndex = applied.StateLogEntryId;
this.LastCommittedIndexTerm = applied.StateLogEntryTerm;
if (lstCommited.Count > 0)
{
this.rn.Commited();
}
//this.rn.Commited(applied.StateLogEntryId);
return(eEntryAcceptanceResult.Committed);
}
}
return(eEntryAcceptanceResult.Accepted);
}
/// <summary>
/// +
/// Can be null.
/// Must be called inside of operation lock.
/// </summary>
/// <param name="logEntryId"></param>
/// <param name="LeaderTerm"></param>
/// <returns></returns>
public StateLogEntrySuggestion GetNextStateLogEntrySuggestionFromRequested(StateLogEntryRequest req)
{
StateLogEntrySuggestion le = new StateLogEntrySuggestion()
{
LeaderTerm = rn.NodeTerm
};
int cnt = 0;
StateLogEntry sle = null;
ulong prevId = 0;
ulong prevTerm = 0;
using (var t = this.rn.db.GetTransaction())
{
if (req.StateLogEntryId == 0)// && req.StateLogEntryTerm == 0)
{
if (rn.entitySettings.DelayedPersistenceIsActive && sleCache.Count > 0)
{
sle = sleCache.OrderBy(r => r.Key).First().Value.Item2;
}
else
{
var trow = t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
new byte[] { 1 }.ToBytes(ulong.MinValue, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true).FirstOrDefault();
if (trow != null && trow.Exists)
{
sle = StateLogEntry.BiserDecode(trow.Value);
}
}
//else
//{
// //should not normally happen
//}
if (sle != null)
{
le.StateLogEntry = sle;
if (
LastCommittedIndexTerm >= le.StateLogEntry.Term
&&
LastCommittedIndex >= le.StateLogEntry.Index
)
{
le.IsCommitted = true;
}
cnt = 2;
}
}
else
{
Tuple <ulong, StateLogEntry> sleTpl;
bool reForward = true;
if (rn.entitySettings.DelayedPersistenceIsActive &&
sleCache.TryGetValue(req.StateLogEntryId, out sleTpl)
)
{
cnt++;
sle = sleTpl.Item2;
prevId = sle.Index;
prevTerm = sle.Term;
if (sleCache.TryGetValue(req.StateLogEntryId + 1, out sleTpl))
{
cnt++;
sle = sleTpl.Item2;
le.StateLogEntry = sle;
}
}
else
{
reForward = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
//new byte[] { 1 }.ToBytes(req.StateLogEntryId, req.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(req.StateLogEntryId, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true).Take(2))
{
cnt++;
sle = StateLogEntry.BiserDecode(el.Value);
if (cnt == 1)
{
prevId = sle.Index;
prevTerm = sle.Term;
}
else
{
le.StateLogEntry = sle;
}
}
}
if (cnt < 2 && reForward)
{
ulong toAdd = (ulong)cnt;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(tblStateLogEntry,
//new byte[] { 1 }.ToBytes(req.StateLogEntryId + toAdd, req.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(req.StateLogEntryId + toAdd, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true).Take(2))
{
cnt++;
sle = StateLogEntry.BiserDecode(el.Value);
if (cnt == 1)
{
prevId = sle.Index;
prevTerm = sle.Term;
}
else
{
le.StateLogEntry = sle;
}
}
}
if (cnt == 2)
{
le.StateLogEntry.PreviousStateLogId = prevId;
le.StateLogEntry.PreviousStateLogTerm = prevTerm;
if (
LastCommittedIndexTerm >= le.StateLogEntry.Term
&&
LastCommittedIndex >= le.StateLogEntry.Index
)
{
le.IsCommitted = true;
}
}
}
}
//if (first)
if (cnt != 2)
{
return(null);
}
return(le);
}
/// <summary>
/// +
/// Can be null.
/// Must be called inside of operation lock.
/// get local commit history for follower
/// </summary>
/// <param name="logEntryId"></param>
/// <param name="LeaderTerm"></param>
/// <returns></returns>
public StateLogEntrySuggestion GetNextStateLogEntrySuggestion(StateLogEntryRequest req)
{
StateLogEntrySuggestion le = new StateLogEntrySuggestion()
{
LeaderTerm = statemachine.NodeTerm
};
int cnt = 0;
StateLogEntry sle = null;
ulong prevId = 0;
ulong prevTerm = 0;
using (var t = this.db.GetTransaction())
{
if (req.StateLogEntryId == 0)// && req.StateLogEntryTerm == 0)
{
var trow = t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(ulong.MinValue, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true).FirstOrDefault();
if (trow != null && trow.Exists)
{
sle = StateLogEntry.BiserDecode(trow.Value);
}
if (sle != null)
{
le.StateLogEntry = sle;
if (
LastCommittedIndexTerm >= le.StateLogEntry.Term
&&
LastCommittedIndex >= le.StateLogEntry.Index
)
{
le.IsCommitted = true;
}
cnt = 2;
}
}
else
{
Tuple <ulong, StateLogEntry> sleTpl;
bool reForward = true;
reForward = false;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
new byte[] { 1 }.ToBytes(req.StateLogEntryId, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true).Take(2))
{
cnt++;
sle = StateLogEntry.BiserDecode(el.Value);
if (cnt == 1)
{
prevId = sle.Index;
prevTerm = sle.Term;
}
else
{
le.StateLogEntry = sle;
}
}
if (cnt < 2 && reForward)
{
ulong toAdd = (ulong)cnt;
foreach (var el in t.SelectForwardFromTo <byte[], byte[]>(stateTableName,
//new byte[] { 1 }.ToBytes(req.StateLogEntryId + toAdd, req.StateLogEntryTerm), true,
new byte[] { 1 }.ToBytes(req.StateLogEntryId + toAdd, ulong.MinValue), true,
new byte[] { 1 }.ToBytes(ulong.MaxValue, ulong.MaxValue), true).Take(2))
{
cnt++;
sle = StateLogEntry.BiserDecode(el.Value);
if (cnt == 1)
{
prevId = sle.Index;
prevTerm = sle.Term;
}
else
{
le.StateLogEntry = sle;
}
}
}
if (cnt == 2)
{
le.StateLogEntry.PreviousStateLogId = prevId;
le.StateLogEntry.PreviousStateLogTerm = prevTerm;
if (
LastCommittedIndexTerm >= le.StateLogEntry.Term
&&
LastCommittedIndex >= le.StateLogEntry.Index
)
{
le.IsCommitted = true;
}
}
}
}
if (cnt != 2)
{
return(null);
}
return(le);
}