public Int32 ScriptMain([In] object[] ScriptParameters, [In] Int32 DefaultReturnCode)
{
int CurrentLatency = (int)ScriptParameters[0];
if (FirstRun)
{
FirstRun = false;
Database.ACR_SetPersistentInt(GetModule(), "ACR_HEALTHMONITOR_STATUS", (int)HealthStatus);
Database.ACR_SetPersistentInt(GetModule(), "ACR_HEALTHMONITOR_VAULT_STATUS", (int)VaultStatus);
LatencyMeasurements.AddRange(new int[LATENCY_MEASUREMENTS_TO_RECORD]);
}
//
// Record the current latency measurement value. If we have a full
// set of history, the process this batch of historical data.
//
if (MeasurementIndex == LATENCY_MEASUREMENTS_TO_RECORD)
{
MeasurementIndex = 0;
ProcessMeasuredLatency();
}
// WriteTimestampedLogEntry(String.Format("LatencyMeasurements[{0}] = {1}", MeasurementIndex, CurrentLatency));
//
// Consider measurement failures as high latency. Usually these
// are due to the measurement timing out entirely (2000ms+) anyway.
//
if (CurrentLatency == -1)
{
CurrentLatency = LATENCY_HIGH_VALUE;
}
LatencyMeasurements[MeasurementIndex++] = CurrentLatency;
return(DefaultReturnCode);
}
/// <summary>
/// This method is called when we have a batch of latency measurements
/// available. Its purpose is to determine the current server health
/// status, schedule a database update, and perform adjustment actions
/// as necessary.
/// </summary>
private void ProcessMeasuredLatency()
{
LATENCY_HEALTH_STATUS NewStatus;
VAULT_HEALTH_STATUS NewVaultStatus;
int MedianLatency;
int VaultLatency;
uint ModuleObject = GetModule();
//
// Find the median latency for the past several measurements and
// set the health status of the server based on that.
//
LatencyMeasurements.Sort();
MedianLatency = LatencyMeasurements[LATENCY_MEASUREMENTS_TO_RECORD / 2];
if (MedianLatency < LATENCY_LOW_VALUE)
{
NewStatus = LATENCY_HEALTH_STATUS.Healthy;
}
else if (MedianLatency < LATENCY_HIGH_VALUE)
{
NewStatus = LATENCY_HEALTH_STATUS.Warning;
}
else
{
NewStatus = LATENCY_HEALTH_STATUS.Unhealthy;
}
SetGlobalInt("ACR_SERVER_LATENCY_MEDIAN", MedianLatency);
VaultLatency = GetGlobalInt("ACR_VAULT_LATENCY");
if (VaultLatency == -1)
{
VaultLatency = VAULT_LATENCY_HIGH_VALUE;
}
if (VaultLatency < VAULT_LATENCY_HIGH_VALUE)
{
NewVaultStatus = VAULT_HEALTH_STATUS.Healthy;
}
else
{
NewVaultStatus = VAULT_HEALTH_STATUS.Unhealthy;
}
//
// Log a message and update the database if the health status has
// changed.
//
if (HealthStatus != NewStatus)
{
WriteTimestampedLogEntry(String.Format("ACR_HealthMonitor.ProcessMeasuredLatency(): Server health status is now {0} (median latency {1})", NewStatus, MedianLatency));
Database.ACR_SetPersistentInt(ModuleObject, "ACR_HEALTHMONITOR_STATUS", (int)NewStatus);
HealthStatus = NewStatus;
//
// Attempt to record some useful data about what is happening
// on the server when we transition to the unhealthy state.
//
if (HealthStatus == LATENCY_HEALTH_STATUS.Unhealthy)
{
DiagnoseServerHealth();
}
}
if (VaultStatus != NewVaultStatus)
{
WriteTimestampedLogEntry(String.Format("ACR_HealthMonitor.ProcessMeasuredLatency(): Vault connection health status is now {0} (latency {1})", NewVaultStatus, VaultLatency));
Database.ACR_SetPersistentInt(ModuleObject, "ACR_HEALTHMONITOR_VAULT_STATUS", (int)NewVaultStatus);
VaultStatus = NewVaultStatus;
}
//
// If backoff for the GameObjUpdate time is enabled, then try and
// adjust module performance.
//
if (GetLocalInt(ModuleObject, "ACR_HEALTHMONITOR_GAMEOBJUPDATE_BACKOFF") != FALSE)
{
int GameObjUpdateTime = SystemInfo.GetGameObjUpdateTime(this);
bool SetTime = false;
uint Now = (uint)Environment.TickCount;
switch (HealthStatus)
{
case LATENCY_HEALTH_STATUS.Healthy:
//
// If we are healthy, then keep adjusting the update
// time downwards until we hit the default value. This
// trades performance for responsiveness.
//
if (GameObjUpdateTime != SystemInfo.DEFAULT_GAMEOBJUPDATE_TIME)
{
if (GameObjUpdateAdjustDelay == 0)
{
LastLatencyAdjustTick = Now;
GameObjUpdateAdjustDelay = LatencyAdjustHysteresis.GetNextDelayTime();
WriteTimestampedLogEntry(String.Format("ACR_HealthMonitor.ProcessMeasuredLatency(): Queued downwards GameObjUpdateTime adjustment in {0} milliseconds (current update time value is {0})...", GameObjUpdateAdjustDelay, GameObjUpdateTime));
}
}
break;
case LATENCY_HEALTH_STATUS.Warning:
//
// Don't take any action if we are in the warning
// state; instead, keep the current update time where
// it is right now, to help avoid thrashing.
//
break;
case LATENCY_HEALTH_STATUS.Unhealthy:
//
// If we are unhealthy, then keep adjusting the update
// time upwards until we hit the maximum value. This
// trades responsiveness for performance.
//
if (GameObjUpdateTime != SystemInfo.MAX_RECOMMENDED_GAMEOBJUPDATE_TIME)
{
SetTime = true;
GameObjUpdateTime += GAMEOBJUPDATE_ADJUSTMENT;
}
break;
}
if (GameObjUpdateAdjustDelay != 0 &&
(Now - LastLatencyAdjustTick) >= GameObjUpdateAdjustDelay &&
GameObjUpdateTime != SystemInfo.DEFAULT_GAMEOBJUPDATE_TIME &&
HealthStatus == LATENCY_HEALTH_STATUS.Healthy)
{
//
// We have passed the required minimum time for a downwards
// latency adjustment, effect it now.
//
SetTime = true;
GameObjUpdateTime -= GAMEOBJUPDATE_ADJUSTMENT;
GameObjUpdateAdjustDelay = 0;
}
if (SetTime)
{
WriteTimestampedLogEntry(String.Format("ACR_HealthMonitor.ProcessMeasuredLatency(): Adjusting GameObjUpdateTime to {0}...", GameObjUpdateTime));
SystemInfo.SetGameObjUpdateTime(this, GameObjUpdateTime);
}
}
}
