private void checkedListBox1_SelectedIndexChanged(object sender, EventArgs e)
{
ck_list_servers.Items.Clear();
foreach (var del in checkedListBox1.CheckedItems)
{
string account = del.ToString().Trim();
DelegateServers deleg = currentDelegateServers.Where(s => s.account.username == account).FirstOrDefault();
foreach (Servers server in deleg.servers)
{
ck_list_servers.Items.Add(server.serverIP + ":" + server.serverPort + " " + server.serverName);
// lb_servers.Text += "\r\nProcessing server " + server.serverName + " " + server.serverIP + ":" + server.serverPort.ToString(); ;
}
}
}
private void bt_account_add_Click(object sender, EventArgs e)
{
DelegateServers delegateAccount = new DelegateServers();
delegateAccount.account = new Accounts();
delegateAccount.account.enableMonitor = false;
delegateServers.Add(delegateAccount);
BindAccounts();
AccountEdit edit = new AccountEdit(delegateAccount.account);
edit.ShowDialog();
dataGrid_accounts.Refresh();
}
private void bt_add_server_Click(object sender, EventArgs e)
{
if (dataGrid_accounts.SelectedRows.Count == 1)
{
var sel = dataGrid_accounts.SelectedRows[0].DataBoundItem as Accounts;
var currentDelegate = new DelegateServers();
foreach (DelegateServers s in delegateServers)
{
if (s.account.username == sel.username)
{
currentDelegate = s;
break;
}
}
if (currentDelegate.servers == null)
{
currentDelegate.servers = new List <Servers>();
}
Servers newServer = new Servers();
newServer.serverName = "new server";
currentDelegate.servers.Add(newServer);
BindAccounts();
BindServers();
ServersEdit edit = new ServersEdit(newServer);
edit.ShowDialog();
dataGrid_servers.Refresh();
}
else
{
MessageBox.Show("must select account!");
}
}
private bool ChooseBestServerToForge(DelegateServers account, out string error, out string tracing)
{
bool res = false;
error = "";
tracing = "ChooseBestServerToForge : Actions Taken: \r\n";
List <Servers> activeServers = account.servers.Where(s => s.isEnable == true).ToList();
Servers mainServer = activeServers.Where(s => s.isMainServer == true).FirstOrDefault();
List <Servers> backupServers = activeServers.Where(s => s.isMainServer == false &&
s.isRebuilding == false && s.blockDiff < 5).OrderByDescending(s => s.consensus).ThenBy(s => s.priority).ToList();
//When not synching, consensus is usually high, so you need to add the -Z flag.
//This way LiskAK will ignore consensus on nodes showing syncing = false.
// switch main server to another one if lastblock > 20
if (account.lastServerSwitch == null || account.lastServerSwitch == new DateTime())
{
account.lastServerSwitch = DateTime.Now.AddMinutes(-60);
}
double minPassedSinceLastSwitch = (DateTime.Now - account.lastServerSwitch).TotalMinutes;
int comsemsusShift = 51;
var bestServersBroadhashConsensus = backupServers.Where(s => s.consensus >= comsemsusShift && s.blockDiff < 4 && s.isRebuilding == false).OrderByDescending(s => s.consensusAvg).ThenByDescending(s => s.consensus).ThenBy(s => s.priority).ToList();
bool isMainServerConsensusSmallerThanBackUp = true;
if (bestServersBroadhashConsensus.Count() > 1)
{
isMainServerConsensusSmallerThanBackUp = (mainServer.consensus < bestServersBroadhashConsensus.Max(s => s.consensusFromLog))?true:false;
}
if ((mainServer.consensus == 0 && bestServersBroadhashConsensus.Count > 0) ||
(mainServer.forgingPositionCurrenSlot == -1 && mainServer.blockDiff >= 5 && bestServersBroadhashConsensus.Count > 0) ||
((mainServer.forgingPositionCurrenSlot == -1 && mainServer.consensus < comsemsusShift &&
minPassedSinceLastSwitch >= 1 && bestServersBroadhashConsensus.Count > 0 &&
isMainServerConsensusSmallerThanBackUp) //2
&& (mainServer.LastBlockMinutsPassedSince > 6))
)
{
//new if
int consensusFromLog = bll.ReadConsensusFromLog(mainServer);
if (consensusFromLog < comsemsusShift || mainServer.consensus < comsemsusShift)
{
#region switch server
account.lastServerSwitch = DateTime.Now;
tracing += "ChooseBestServerToForge :switch main server : \r\n" + mainServer.serverName + " BlockDiff:" + mainServer.blockDiff + " TO " + bestServersBroadhashConsensus[0].serverName;
string previousMain = mainServer.serverName;
string oldConsensus = mainServer.consensus.ToString();
mainServer.isMainServer = false;
//choose best backupserver
string sms = "Now Forging:" + bestServersBroadhashConsensus[0].serverName + " - " + " Swicth forged server from: " + previousMain + " With Consensus:" + oldConsensus + " to: "
+ bestServersBroadhashConsensus[0].serverName
+ " With Consensus:" + bestServersBroadhashConsensus[0].consensus.ToString() + " Average Consensus:" + bestServersBroadhashConsensus[0].consensusAvg.ToString();
//disable forging main
string err = "";
bll.ForgingActionSSH(false, mainServer, account.account.passPhrase, out err);
mainServer.isMainServer = false;
//enabe forging backup
string err2 = "";
bll.ForgingActionSSH(true, bestServersBroadhashConsensus[0], account.account.passPhrase, out err2);
bestServersBroadhashConsensus[0].isMainServer = true;
//RELOAD COLECTIONS
mainServer = activeServers.Where(s => s.isMainServer == true).FirstOrDefault();
backupServers = activeServers.Where(s => s.isMainServer == false && s.isChainSynced == true && s.isRebuilding == false).OrderBy(s => s.blockDiff).ToList();
string tailsLogs = "\n" + mainServer.serverName + "\nTail Log:" + mainServer.tailLog + "\n";
bll.SendEmail(sms, account.account.email, bll.GetAccountString(account) + tailsLogs + "\nERROR:" + err + " " + err2);
#endregion
}
}
if (mainServer != null && mainServer.isRebuilding)
{
error += "Warning: MainServer is rebuilding...\r\n";
}
foreach (Servers server in backupServers)
{
if (server.isRebuilding)
{
error += "Warning: BackupServer is rebuilding...\r\n";
}
}
//if theres a backup
if (mainServer != null && backupServers != null && backupServers.Count > 0)
{
tracing += " There's MainServer and BackUpServer \r\n";
//mainServer.isChainSynced try activate mainserver
if (mainServer.isChainSynced)
{
//disable forging in backup
foreach (Servers server in backupServers)
{
string error1 = "";
bool resDisable = bll.ForgingActionSSH(false, server, account.account.passPhrase, out error1);
error += error1;
}
if (true)//enable forging main
{
string error2 = "";
bool resEnable = bll.ForgingActionSSH(true, mainServer, account.account.passPhrase, out error2);
error += error2;
res = resEnable;
tracing += " MainServer Acivate Forging, disable in BackUpServer \r\n";
}
} //mainServer.isChainSynced==false try activate best backupserver
else if (mainServer.isChainSynced == false && backupServers != null && backupServers.Count > 0)
{
//disable forging in main
string error1 = "";
bool resDisable = bll.ForgingActionSSH(false, mainServer, account.account.passPhrase, out error1);
error += error1;
//verify forging
List <Servers> backServerForging = backupServers.Where(s => s.isChainSynced == true)
.OrderByDescending(s => s.consensusAvg).OrderByDescending(s => s.consensus).ToList();
//if theres only one forging
if (backServerForging != null && backServerForging.Count == 1)
{
string error2 = "";
bool resEnable = bll.ForgingActionSSH(true, backServerForging[0], account.account.passPhrase, out error2);
error += error2;
res = resEnable;
if (res)
{
tracing += " BackUpServer: " + backServerForging[0].serverName + " Activate Forging, MainServer is not synced \r\n";
}
}
//if theres more than one forging correct fork
if (backServerForging != null && backServerForging.Count > 1)
{
Servers serverToForge = backupServers.Where(s => s.isChainSynced).FirstOrDefault();
//disable all minus elected
foreach (Servers s in backServerForging)
{
if (s.serverIP != serverToForge.serverIP)
{
string error3 = "";
bll.ForgingActionSSH(false, s, account.account.passPhrase, out error3); //###
error += error3;
}
}
//enable best serverToForge
string error2 = "";
bool resEnable = bll.ForgingActionSSH(true, serverToForge, account.account.passPhrase, out error2);
error += error2;
res = resEnable;
if (res)
{
tracing += " BackUpServer: " + serverToForge + " Activate Forging, MainServer is not synced \r\n";
}
}
}
}
//if theres only one
if (mainServer.isChainSynced && (backupServers == null || backupServers.Count == 0))
{
tracing += " There's Only MainServer \r\n";
string error2 = "";
bool resEnable = bll.ForgingActionSSH(true, mainServer, account.account.passPhrase, out error2);
error += error2;
res = resEnable;
if (res)
{
tracing += " mainServer Activate Forging \r\n";
}
}
//reload one at a time backups if forging last block <3 min and lastReboot time passed >2 hours
if (mainServer.LastBlockMinutsPassedSince <= 2)
{
var backs = account.servers.Where(s => s.isMainServer == false && s.isRebuilding == false && s.lastReboot.AddHours(5) <= DateTime.Now).OrderBy(s => s.lastReboot).Take(1).ToList();
if (backs.Count > 0)
{
string[] serv = backs.Select(s => s.serverName).ToArray();
string serversToReload = string.Join(",", serv);
bll.SendEmail("Try Reload Nº backups Servers:" + backs.Count.ToString() + " Servers Reloaded :" + serversToReload, currentConfig.emailto, " Servers Reloaded :" + serversToReload);
foreach (Servers bck in backs)
{
bck.lastReboot = DateTime.Now;
bll.StopAndStartNodeSSH(bck);
}
}
}
return(res);
}
private string TakeActions(DelegateServers account)
{
string tracing = "";
List <Servers> activeServers = account.servers.Where(s => s.isEnable == true).OrderBy(s => s.priority).ToList();
int numberServersAvailable = activeServers.Where(s => s.blockDiff < blockDiffToRebuild).Count();
foreach (Servers servers in activeServers)
{
string fork = ("Fork - {'delegate':'" + account.account.generatorPublicKey.Trim() + "'").Replace("'", "\"");
List <string> forkLogs = servers.tailLog.Split('\n').ToList();
//"cause":5
bool rebuildNowForkIssue = false;
foreach (string logLine in forkLogs)
{
if (logLine.Contains(fork))
{
if (logLine.Contains("5"))
{
rebuildNowForkIssue = true;
bll.SendEmail(servers.serverName + " Fork detected rebuild will be done", account.account.email, "tailLog:" + servers.tailLog);
break;
}
}
}
#region rebuild
if ((servers != null && servers.isChainSynced == false && servers.isRebuilding == false && servers.blockDiff > blockDiffToRebuild) ||
(servers != null && servers.isRebuilding == false &&
servers.isForging == true && servers.LastBlockMinutsPassedSince > 120 && account.account.rate <= 101 && numberServersAvailable > 1) ||
(servers != null && servers.tailLog.Contains("SIGKILL")) ||
(servers != null && servers.tailLog.Contains("FATAL ERROR"))
//|| (servers != null && servers.tailLog.Contains("Failed to add vote"))
|| (servers != null && servers.tailLog.Contains("SIGABRT")) ||
(servers != null && servers.tailLog.Contains("FATAL")) ||
(servers != null && rebuildNowForkIssue) ||
(servers != null && servers.tailLog.Contains("Rebuilding blockchain, current block height: 1")) ||
servers.isRebuilding == true && servers.lastRebuild.AddMinutes(12) < DateTime.Now
//Rebuilding blockchain, current block height: 1
)
{
servers.consensus = 0;
//if is main server switch emidialty
if (servers.isMainServer && servers.isForging)
{
string main = servers.serverName;
string log = servers.tailLog;
var newMain = activeServers.Where(s => s.isMainServer == false && s.blockDiff < 2).OrderByDescending(s => s.consensus).ThenBy(s => s.priority).FirstOrDefault();
if (newMain != null)
{
string err = "";
//disable forging on main
bll.ForgingActionSSH(false, servers, account.account.passPhrase, out err);
servers.isForging = false;
servers.isMainServer = false;
//enable forgin onback
bll.ForgingActionSSH(true, newMain, account.account.passPhrase, out err);
newMain.isForging = true;
newMain.isMainServer = true;
bll.SendEmail("Priority Swith Server to " + newMain.serverName + " Main will rebuild:" + main, account.account.email, servers.serverName + ": TailLog:" + log);
}
}
servers.lastRebuild = DateTime.Now;
servers.lastReboot = DateTime.Now;
numberServersAvailable = numberServersAvailable - 1;
tracing += "Rebuild SERVER " + servers.serverName + ": \r\n";
string rebuild = bll.RebuildNodeClosePortSSH(servers, ck_allowrebuild.Checked);
if (rebuild.Contains("Failed"))
{
System.Threading.Thread.Sleep(1000);
//retry one more time
rebuild += " Rebuild Failed, retry rebuild :" + bll.RebuildNodeClosePortSSH(servers, ck_allowrebuild.Checked);
}
tracing += rebuild;
servers.isRebuilding = true;
servers.isChainSyncing = true;
servers.isChainSynced = false;
servers.lastRebuild = DateTime.Now;
servers.isForging = false;
string serverResult = bll.GetAccountString(account);//GetServerJsonToNotify(servers);
bll.SendEmail("SERVER START REBUILD " + servers.serverName, account.account.email, "SERVER START REBUILD " + servers.serverName + " " + DateTime.Now.ToString() + "\nservers.blockDiff:" + servers.blockDiff + " server data: " + serverResult + " Tracing: " + tracing + " TailLog:" + servers.tailLog);
}
#endregion
//open port
if (servers.isRebuilding == false && servers.blockDiff < blockDiffToRebuild)
{
bll.OpenPortSSH(servers);
}
}
if (activeServers.Count() > 0)// can only have one server forging
{
string error = "";
tracing = "";
bool resBestServerToForge = ChooseBestServerToForge(account, out error, out tracing);
if (!resBestServerToForge)
{
tracing += "\r\nMessage: " + error;
}
}
return(tracing);
}
private Servers GetServerMonitorData_V2(Servers server, DelegateServers account)
{
string name = server.serverName;
string error = "";
server.tailLog = string.Empty;
server.error = string.Empty;
var client = bll.GetConection(server);//new SshClient(server.serverIP, server.userName, server.userPassword);
try
{
bool isLiskRunning = bll.VerifyIfLiskIsRunning_V2(server, out error, client);
decimal missedBlockPrevous = account.account.missedblocks;
Servers serverPreviousData = (Servers)server.Clone();
string url = "http://" + server.serverIP + ":" + server.serverPort;
BlockChainStatus status = bll.GetBlockChainStatusSSH_V2(server, out error, client);
server.error += error;
if (status.success && isLiskRunning)
{
server.blockChainHeight = status.height;
server.blockDiff = status.blocks;
server.isChainSyncing = status.syncing;
server.broadhash = status.broadhash;
server.consensus = string.IsNullOrEmpty(status.consensus) ? 0 : Convert.ToInt16(status.consensus);
// server.BroadhashConsensus100=status
if (server.isChainSyncing)
{
server.isChainSynced = false;
server.isRebuilding = true;
//server.lastRebuild = DateTime.Now;
if (serverPreviousData.isForging)
{
server.isForging = false;
}
}
else
{
server.isChainSynced = true;
server.isRebuilding = false;
}
}
//GetMaxPeerBlock
string erro2 = "";
//server.maxPeerBlock= bll.GetMaxPeerBlock(out erro2,server.serverPort);
server.maxPeerBlock = bll.GetMaxPeerBlock_V2(out erro2, server, client);
server.error += erro2;
//blockdiff
server.blockDiff = server.maxPeerBlock - server.blockChainHeight;
if (server.blockDiff < 0)
{
server.blockDiff = server.blockDiff * -1;
}
if (server.blockDiff > blockDiffToRebuild)
{
server.isChainSynced = false;
if (serverPreviousData.isRebuilding == true && server.isRebuilding == false /*changed because GetBlockChainStatusSSH above has issue*/)
{
server.isChainSynced = false;
server.isRebuilding = true;
}
}
else
{
server.isChainSynced = true;
server.isRebuilding = false;
}
//latestBlockForged
string error3 = "";
if (server.isChainSynced == true && server.isRebuilding == false)
{
Block block = bll.GetLatestBlockSSH_V2(server, account.account.generatorPublicKey, out error3, client);
server.LastBlockMinutsPassedSince = block.minutsPassedSinceLastBlock;
server.LastBlockForgedHeight = block.height;
// getnextforgers
int position = bll.GetForgingPosition(server, account.account.generatorPublicKey, client);
server.forgingPositionCurrenSlot = position == -1?-1:position + 1;
}
else
{
Block block = new Block();
server.LastBlockMinutsPassedSince = 9999;
server.LastBlockForgedHeight = 0;
server.forgingPositionCurrenSlot = -1;
}
server.error += error3;
server.tailLog = bll.ReadTailLog_V2(server, client);
////notify END rebuild
if (serverPreviousData.isChainSynced == false && serverPreviousData.isRebuilding && server.isRebuilding == false)
{
string serverResult = bll.GetAccountString(account);//GetServerJsonToNotify(server);
if (serverPreviousData.isRebuilding == true && server.isRebuilding == false)
{
//server.lastRebuild = DateTime.Now.AddMinutes(10);
File.WriteAllText(AppDomain.CurrentDomain.BaseDirectory + "\\logs\\" + "REBUILD_END_" + server.serverName + "_" + DateTime.Now.Ticks.ToString() + ".txt", "Rebuild End at " + DateTime.Now.ToString() + " \r\n Server Data:" + serverResult);
bll.SendEmail("SERVER END REBUILD " + server.serverName + " " + DateTime.Now.ToString(), account.account.email, "SERVER END REBUILD " + server.serverName + " " + DateTime.Now.ToString() + " server data: " + serverResult);
}
}
//notify LastBlockForgedHeight
if (server.isForging && server.isChainSynced && numberTickesMade > 1)
{
if (server.LastBlockLastNofication < DateTime.Now)
{
if (account.account.rate <= 101 && serverPreviousData.LastBlockForgedHeight < server.LastBlockForgedHeight && server.LastBlockForgedHeight > 0)
{
//string accountString = JsonConvert.SerializeObject(account);
string accountString = bll.GetAccountString(account);
string sms = account.account.username + " Rank: " + account.account.rate.ToString() + " Last Block: " + DateTime.Now.ToString() + " - " + server.serverName + " LastBlockForgedHeight:" + server.LastBlockForgedHeight.ToString() + " LastBlockMinutsPassed: " + server.LastBlockMinutsPassedSince + " this notification is sent every 5 hours, see inside account data.!!!";
bll.SendEmail(sms, account.account.email, sms + " Account: " + accountString);
}
else
{
if (account.account.rate > 101)
{
//string accountString = JsonConvert.SerializeObject(account);
string accountString = bll.GetAccountString(account);
string sms = account.account.username + " Rank: " + account.account.rate.ToString() + " this notification is sent every 5 hours, see inside account data.!!!";
bll.SendEmail(sms, account.account.email, sms + " Account:" + accountString);
}
}
server.LastBlockLastNofication = DateTime.Now.AddHours(5);
}
}
//notify missed block
if (missedBlockPrevous < account.account.missedblocks)
{
string accountString = bll.GetAccountString(account); //JsonConvert.SerializeObject(account);
string sms = "Missed Block Notification " + account.account.username + " Rank:" + account.account.rate.ToString() + " " + DateTime.Now.ToString() + " - Server: " + server.serverName + " Missed Blocks:" + account.account.missedblocks;
bll.SendEmail(sms, account.account.email, sms + " Account:" + accountString);
}
}
catch
{
}
finally
{
//client.Disconnect();
//client.Dispose();
}
return(server);
}
private string PerformMonitor(DelegateServers account)
{
StringBuilder message = new StringBuilder();;
string takeactions_result = "";
try
{
List <Task> tasks = new List <Task>();
foreach (Servers s in account.servers.Where(s => s.isEnable == true).OrderByDescending(s => s.isForging).ThenBy(s => s.priority))
{
if (s.lastRebuild == null || s.lastRebuild == DateTime.Now)
{
s.lastRebuild = DateTime.Now;
}
if (s.lastReboot == null || s.lastReboot == DateTime.Now)
{
s.lastReboot = DateTime.Now.AddHours(-5);
}
// GetServerMonitorData_V2(s, account);
Task t = Task.Factory.StartNew(() =>
{
GetServerMonitorData_V2(s, account);
});
tasks.Add(t);
}
Task.WaitAll(tasks.ToArray());
if (numberTickesMade > 0)
{
takeactions_result = TakeActions(account);
}
// WriteLog("\n\r StartDoMonitor servers account " + s.account.username + " Rank:" + s.account.rate + "\n\r");
//print output
#region PRINT OUT PUT
message.Append("\r\n############## PROCESSING ACCOUNT " + account.account.username.ToUpper() + " #####################\r\n");
string serverResult = bll.GetAccountString(account);//GetServerJsonToNotify(s);//JsonConvert.SerializeObject(s);
message.Append("\r\nACCOUNT INFO:\r\n" + serverResult);
// message.Append("\r\n");
foreach (Servers s in account.servers.Where(s => s.isEnable == true).OrderByDescending(s => s.isForging).OrderByDescending(s => s.consensus))
{
message.Append("\r\nSERVER INFO: " + s.priority + " " + s.serverName.ToUpper() + " " + s.serverIP + " " + s.serverPort + " IS MAINSERVER:" + s.isMainServer + " IS FORGING:" + s.isForging + " IS REBUILDING:" + s.isRebuilding + "\r\n## BLOCK DIFF: " + s.blockDiff + " ##\r\n");
message.Append("Consensus: " + s.consensus.ToString() + " ConsensusFromLog: " + s.consensusFromLog.ToString() + "\r\n");
message.Append("ConsensusStr: " + s.consensusStr.Trim() + "\r\n");
message.Append("ConsensusAVG: " + s.consensusAvg.ToString() + "\r\n");
message.Append("isChainSyncing: " + s.isChainSyncing.ToString() + "\r\n");
message.Append("Forging Position Current Slot: " + s.forgingPositionCurrenSlot.ToString() + "\r\n");
message.Append("Estimated Time(sec) To Forge : " + (s.forgingPositionCurrenSlot == -1?">100 seg" : (s.forgingPositionCurrenSlot * 10).ToString() + " seg") + "\r\n");
message.Append("Last Reboot: " + s.lastReboot.ToString() + "\r\n");
if (s.isRebuilding)
{
message.AppendLine(s.serverName.ToUpper() + "\r\n ..IS REBUILDING....\r\n");
}
//string serverResult = bll.GetAccountString(account);//GetServerJsonToNotify(s);//JsonConvert.SerializeObject(s);
//message.Append("\r\nACCOUNT INFO:\r\n" + serverResult);
if (s.error != string.Empty)
{
message.Append("\r\nERROR - " + s.error);
}
message.Append("\r\n");
}
message.Append("\r\n" + takeactions_result);
#endregion
}
catch (Exception ex)
{
string error = "EXCEPTION Monitor " + account.account.username + DateTime.Now.ToString() + " EXCEPTION:" + ex.Message;
if (ex.InnerException != null)
{
error += ex.InnerException + ex.StackTrace;
}
message.Append("EXCEPTION Monitor " + account.account.username + DateTime.Now.ToString() + " EXCEPTION:" + ex.Message);
bll.SendEmail("ERROR DoMonitor " + account.account.username + DateTime.Now.ToString(), emailto, error);
}
return(message.ToString());;
}
