public async Task AuditorStateTest(KeyPair clientKeyPair, ConnectionState state, Type excpectedException)
{
context.AppState.State = ApplicationState.Rising;
var clientConnection = new AlphaWebSocketConnection(context, new FakeWebSocket(), "127.0.0.1")
{
ClientPubKey = clientKeyPair.PublicKey,
ConnectionState = state
};
var envelope = new AuditorState
{
PendingQuanta = new List <MessageEnvelope>(),
State = ApplicationState.Running
}.CreateEnvelope();
envelope.Sign(clientKeyPair);
using var writer = new XdrBufferWriter();
var inMessage = envelope.ToIncomingMessage(writer);
await AssertMessageHandling(clientConnection, inMessage, excpectedException);
if (excpectedException == null)
{
Assert.AreEqual(context.AppState.State, ApplicationState.Running);
}
}
private bool AddQuanta(RawPubKey pubKey, AuditorState currentState, AuditorState newAuditorState)
{
if (!currentState.HasMorePendingQuanta ||
newAuditorState.HasMorePendingQuanta && newAuditorState.PendingQuanta.Count < 1) //prevent spamming
{
logger.Trace($"Unable to add auditor's {((KeyPair)pubKey).AccountId} quanta.");
currentState.HasMorePendingQuanta = false;
return(false);
}
currentState.HasMorePendingQuanta = newAuditorState.HasMorePendingQuanta;
var lastAddedApex = currentState.PendingQuanta.LastOrDefault()?.Message.MessageId ?? -1;
var alphaPubkey = (RawPubKey)Context.Settings.KeyPair.PublicKey;
foreach (var envelope in newAuditorState.PendingQuanta)
{
var currentQuantum = (Quantum)envelope.Message;
if (lastAddedApex != -1 && currentQuantum.Apex != lastAddedApex + 1)
{
return(false);
}
lastAddedApex = currentQuantum.Apex;
if (envelope.Signatures.All(s => !s.Signer.Equals(alphaPubkey)) || !envelope.AreSignaturesValid())
{
return(false);
}
currentState.PendingQuanta.Add(envelope);
}
logger.Trace($"Auditor's {((KeyPair)pubKey).AccountId} quanta added.");
return(true);
}
public async Task AuditorStateTest(KeyPair clientKeyPair, ConnectionState state, Type excpectedException)
{
Global.AppState.State = ApplicationState.Rising;
var clientConnection = new AlphaWebSocketConnection(new FakeWebSocket(), "127.0.0.1")
{
ClientPubKey = clientKeyPair.PublicKey,
ConnectionState = state
};
var envelope = new AuditorState
{
PendingQuanta = new List <MessageEnvelope>(),
State = ApplicationState.Running
}.CreateEnvelope();
envelope.Sign(clientKeyPair);
await AssertMessageHandling(clientConnection, envelope, excpectedException);
if (excpectedException == null)
{
Assert.AreEqual(Global.AppState.State, ApplicationState.Running);
}
}
private void OnAuditorStateChanged(AuditorState oldState, AuditorState newState, Auditor auditor)
{
if (newState == AuditorState.Completed)
{
//int removedReports = Services.SecretaryService.GetReportInfo(auditor.GeneratorId).RemoveReports(auditor.Count);
//UDBG.Log($"reports handled count => {removedReports}");
}
}
private void UpdateState(AuditorState state)
{
switch (state)
{
case AuditorState.MoveToLoad: {
//progressParent.Deactivate();
//countText.Deactivate();
progressParent.Activate();
countText.Activate();
rectTransform.localScale = new Vector3(-1, 1, 1);
countText.GetComponent <RectTransform>().localScale = new Vector3(-1, 1, 1);
animator.SetTrigger(animNames[Direction.Right]);
rectTransform.anchoredPosition = Vector2.Lerp(rightPosition, leftPosition, Auditor.NormalizedTimer);
}
break;
case AuditorState.Loading: {
animator.SetTrigger(animNames[Direction.StayLeft]);
progressParent.Activate();
countText.Activate();
rectTransform.localScale = Vector3.one;
countText.GetComponent <RectTransform>().localScale = Vector3.one;
rectTransform.anchoredPosition = leftPosition;
}
break;
case AuditorState.MoveToUnload: {
progressParent.Activate();
countText.Activate();
rectTransform.localScale = Vector3.one;
countText.GetComponent <RectTransform>().localScale = Vector3.one;
animator.SetTrigger(animNames[Direction.Right]);
rectTransform.anchoredPosition = Vector2.Lerp(leftPosition, rightPosition, Auditor.NormalizedTimer);
}
break;
case AuditorState.Unloading: {
progressParent.Activate();
countText.Activate();
animator.SetTrigger(animNames[Direction.StayRight]);
rectTransform.anchoredPosition = rightPosition;
rectTransform.localScale = Vector3.one;
countText.GetComponent <RectTransform>().localScale = Vector3.one;
}
break;
default: {
rectTransform.localScale = Vector3.one;
countText.GetComponent <RectTransform>().localScale = Vector3.one;
animator.SetTrigger(animNames[Direction.StayRight]);
progressParent.Deactivate();
countText.Deactivate();
rectTransform.anchoredPosition = rightPosition;
}
break;
}
}
private void OnAuditorStateChanged(AuditorState oldState, AuditorState newState, Auditor info)
{
if (isInitialized)
{
if (Auditor != null && (Auditor.Id == info.Id))
{
UpdateState(newState);
}
}
}
private void OnAuditorStateChanged(AuditorState oldState, AuditorState newState, Auditor auditor)
{
if (generator != null && (generator.GeneratorId == auditor.GeneratorId))
{
UpdateViews();
if (newState == AuditorState.Completed)
{
RemoveAuditorView(auditor);
}
}
}
private void ChangeState(AuditorState newState)
{
AuditorState oldState = State;
if (State != newState)
{
if (newState == AuditorState.MoveToLoad)
{
timer = 0f;
State = AuditorState.MoveToLoad;
GameEvents.OnAuditorStateChanged(oldState, State, this);
}
else if (newState == AuditorState.Loading)
{
timer -= MoveInterval;
State = AuditorState.Loading;
GameEvents.OnAuditorStateChanged(oldState, State, this);
if (timer >= LoadInterval)
{
ChangeState(AuditorState.MoveToUnload);
}
}
else if (newState == AuditorState.MoveToUnload)
{
timer -= LoadInterval;
State = AuditorState.MoveToUnload;
GameEvents.OnAuditorStateChanged(oldState, State, this);
if (timer >= MoveInterval)
{
ChangeState(AuditorState.Unloading);
}
}
else if (newState == AuditorState.Unloading)
{
timer -= MoveInterval;
State = AuditorState.Unloading;
GameEvents.OnAuditorStateChanged(oldState, State, this);
if (timer > UnloadInterval)
{
ChangeState(AuditorState.Completed);
}
}
else if (newState == AuditorState.Completed)
{
State = AuditorState.Completed;
GameEvents.OnAuditorStateChanged(oldState, State, this);
}
}
}
/// <summary>
/// Checks that all quanta have valid Alpha signature
/// </summary>
private static bool IsStateValid(AuditorState state)
{
var alphaPubkey = (RawPubKey)Global.Settings.KeyPair.PublicKey;
var lastApex = Global.QuantumStorage.CurrentApex;
for (int i = 0; i < state.PendingQuanta.Count; i++)
{
var currentQuantumEnvelope = state.PendingQuanta[i];
var currentQuantum = (Quantum)currentQuantumEnvelope.Message;
if (!(lastApex + 1 == currentQuantum.Apex &&
currentQuantumEnvelope.Signatures.Any(s => s.Signer.Equals(alphaPubkey)) &&
currentQuantumEnvelope.AreSignaturesValid()))
{
return(false);
}
lastApex = currentQuantum.Apex;
}
return(true);
}
public override async Task HandleMessage(AuditorWebSocketConnection connection, IncomingMessage message)
{
var stateRequestMessage = (AuditorStateRequest)message.Envelope.Message;
var hasQuanta = true;
var aboveApex = stateRequestMessage.TargetApex;
var batchSize = 50;
while (hasQuanta)
{
if (!Context.QuantumStorage.GetQuantaBacth(aboveApex + 1, batchSize, out var currentBatch) &&
(aboveApex + 1 < Context.QuantumStorage.CurrentApex))
{
currentBatch = await Context.PersistenceManager.GetQuantaAboveApex(aboveApex, batchSize); //quanta are not found in the in-memory storage
if (currentBatch.Count < 1)
{
throw new Exception("No quanta from database.");
}
}
if (currentBatch == null)
{
currentBatch = new List <MessageEnvelope>();
}
hasQuanta = currentBatch.Count == batchSize;
var state = new AuditorState
{
State = Context.AppState.State,
PendingQuanta = currentBatch,
HasMorePendingQuanta = hasQuanta
};
await connection.SendMessage(state);
var lastQuantum = currentBatch.LastOrDefault();
aboveApex = lastQuantum?.Message.MessageId ?? 0;
}
;
}
public override async Task HandleMessage(AuditorWebSocketConnection connection, MessageEnvelope messageEnvelope)
{
var stateRequestMessage = (AuditorStateRequest)messageEnvelope.Message;
var allPendingQuanta = await Global.PersistenceManager.GetQuantaAboveApex(stateRequestMessage.TargetApex);
var skip = 0;
var maxQuantaPerMessage = 10;
var hasQuanta = true;
while (hasQuanta)
{
var currentBatch = allPendingQuanta.Skip(skip).Take(maxQuantaPerMessage).ToList();
hasQuanta = (skip + maxQuantaPerMessage) < allPendingQuanta.Count;
skip += maxQuantaPerMessage;
var state = new AuditorState
{
State = Global.AppState.State,
PendingQuanta = currentBatch,
HasMorePendingQuanta = hasQuanta
};
await connection.SendMessage(state);
}
;
}
public async Task AddAuditorState(RawPubKey pubKey, AuditorState auditorState)
{
await semaphoreSlim.WaitAsync();
try
{
if (!applyDataTimer.Enabled) //start timer
{
applyDataTimer.Start();
}
logger.Trace($"Auditor state from {((KeyPair)pubKey).AccountId} received by AlphaCatchup.");
if (Context.AppState.State != ApplicationState.Rising)
{
logger.Warn($"Auditor state messages can be only handled when Alpha is in rising state. State sent by {((KeyPair)pubKey).AccountId}");
return;
}
if (!allAuditorStates.TryGetValue(pubKey, out var pendingAuditorState))
{
pendingAuditorState = new AuditorState
{
HasMorePendingQuanta = true,
PendingQuanta = new List <MessageEnvelope>()
};
allAuditorStates.Add(pubKey, pendingAuditorState);
logger.Trace($"Auditor state from {((KeyPair)pubKey).AccountId} added.");
}
if (AddQuanta(pubKey, pendingAuditorState, auditorState)) //check if auditor sent all quanta already
{
if (pendingAuditorState.HasMorePendingQuanta) //wait while auditor will send all quanta it has
{
logger.Trace($"Auditor {((KeyPair)pubKey).AccountId} has more quanta. Timer reseted.");
applyDataTimer.Reset(); //if timer is running reset it. We need to try to wait all possible auditors data
return;
}
logger.Trace($"Auditor {((KeyPair)pubKey).AccountId} state is validated.");
validAuditorStates.Add(pubKey, pendingAuditorState);
}
int majority = Context.GetMajorityCount(),
totalAuditorsCount = Context.GetTotalAuditorsCount();
var completedStatesCount = allAuditorStates.Count(s => !s.Value.HasMorePendingQuanta);
if (completedStatesCount == totalAuditorsCount)
{
await TryApplyAuditorsData();
}
}
catch (Exception exc)
{
logger.Error(exc, "Error on adding auditors state");
Context.AppState.State = ApplicationState.Failed;
}
finally
{
semaphoreSlim.Release();
}
}
public static async Task AddAuditorState(RawPubKey pubKey, AuditorState auditorState)
{
await semaphoreSlim.WaitAsync();
try
{
if (Global.AppState.State != ApplicationState.Rising)
{
throw new InvalidOperationException("Auditor state messages can be only handled when Alpha is in rising state");
}
if (allAuditorStates.TryGetValue(pubKey, out var pendingAuditorState))
{
if (!pendingAuditorState.HasMorePendingQuanta) //check if auditor send all quanta already
{
return;
}
allAuditorStates[pubKey].PendingQuanta.AddRange(auditorState.PendingQuanta);
allAuditorStates[pubKey].HasMorePendingQuanta = auditorState.HasMorePendingQuanta;
}
else
{
allAuditorStates.Add(pubKey, auditorState);
}
var currentAuditorState = allAuditorStates[pubKey];
if (currentAuditorState.HasMorePendingQuanta) //wait while auditor will send all quanta it has
{
return;
}
if (IsStateValid(currentAuditorState))
{
validAuditorStates.Add(pubKey, currentAuditorState);
}
int majority = MajorityHelper.GetMajorityCount(),
totalAuditorsCount = MajorityHelper.GetTotalAuditorsCount();
var completedStatesCount = allAuditorStates.Count(s => !s.Value.HasMorePendingQuanta);
if (completedStatesCount < majority)
{
return;
}
var possibleConsensusCount = (totalAuditorsCount - completedStatesCount) + validAuditorStates.Count;
if (validAuditorStates.Count >= majority)
{
await ApplyAuditorsData();
}
else if (possibleConsensusCount < majority)
{
logger.Error("Majority of auditors are connected, but there is no consensus");
Global.AppState.State = ApplicationState.Failed;
}
}
catch (Exception exc)
{
logger.Error(exc, "Error on adding auditors state");
Global.AppState.State = ApplicationState.Failed;
}
finally
{
semaphoreSlim.Release();
}
}
public static void OnAuditorStateChanged(AuditorState oldState, AuditorState newState, Auditor auditor) => AuditorStateChanged?.Invoke(oldState, newState, auditor);