/// <summary>
/// Gets the node.
/// </summary>
/// <param name="rockContext">The rock context.</param>
/// <param name="nodeName">Name of the node.</param>
/// <returns></returns>
private static WebFarmNode GetNode(RockContext rockContext, string nodeName)
{
var webFarmNodeService = new WebFarmNodeService(rockContext);
var webFarmNode = webFarmNodeService.Queryable().Single(wfn => wfn.NodeName == nodeName);
return(webFarmNode);
}
/// <summary>
/// Gets the node record, potentially creating it.
/// </summary>
/// <param name="rockContext">The rock context.</param>
/// <param name="nodeName">Name of the node.</param>
/// <param name="pollingInterval">The polling interval.</param>
/// <returns>
/// <c>true</c> if [is polling interval in use] [the specified node name]; otherwise, <c>false</c>.
/// </returns>
private static bool IsPollingIntervalInUse(RockContext rockContext, string nodeName, decimal pollingInterval)
{
Debug($"Checking poll interval {pollingInterval}");
var webFarmNodeService = new WebFarmNodeService(rockContext);
var anyMatches = webFarmNodeService.Queryable().Any(wfn =>
wfn.NodeName != nodeName &&
wfn.CurrentLeadershipPollingIntervalSeconds == pollingInterval);
return(anyMatches);
}
/// <summary>
/// Handles the ItemDataBound event of the rNodes control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="System.Web.UI.WebControls.RepeaterItemEventArgs"/> instance containing the event data.</param>
protected void rNodes_ItemDataBound(object sender, System.Web.UI.WebControls.RepeaterItemEventArgs e)
{
var viewModel = e.Item.DataItem as WebFarmNodeService.NodeViewModel;
if (viewModel == null)
{
return;
}
var spanLastSeen = e.Item.FindControl("spanLastSeen");
var lChart = e.Item.FindControl("lChart") as Literal;
var lLastSeen = e.Item.FindControl("lLastSeen") as Literal;
spanLastSeen.Visible = viewModel.IsUnresponsive;
lLastSeen.Text = WebFarmNodeService.GetHumanReadablePastTimeDifference(viewModel.LastSeen);
// Show chart for responsive nodes
if (viewModel.IsActive && !viewModel.IsUnresponsive && viewModel.Metrics.Count() > 1)
{
var samples = WebFarmNodeMetricService.CalculateMetricSamples(viewModel.Metrics, _cpuMetricSampleCount, ChartMinDate, _chartMaxDate);
var html = GetChartHtml(samples);
lChart.Text = html;
}
}
/// <summary>
/// Does the leadership poll.
/// </summary>
internal static async Task DoLeadershipPollAsync()
{
// If another node pinged this node, then that node is the leader, not this one
if (_wasPinged)
{
Debug("My time to poll. I was pinged, so I'm not the leader");
_wasPinged = false;
return;
}
Debug("My time to poll. I was not pinged, so I'm starting leadership duties");
// Ping other nodes with a unique key for this ping-pong round
var pollingTime = RockDateTime.Now;
_leadershipPingKey = Guid.NewGuid();
PublishEvent(EventType.Ping, payload: _leadershipPingKey.Value.ToString());
// Assert this node's leadership in the database
using (var rockContext = new RockContext())
{
var webFarmNodeService = new WebFarmNodeService(rockContext);
var nodes = webFarmNodeService.Queryable().ToList();
var thisNode = nodes.FirstOrDefault(wfn => wfn.NodeName == NodeName);
var otherNodes = nodes.Where(wfn => wfn.NodeName != NodeName);
if (!thisNode.IsLeader)
{
AddLog(rockContext, WebFarmNodeLog.SeverityLevel.Info, _nodeId, EventType.Availability, $"{NodeName} assumed leadership");
}
thisNode.IsLeader = true;
thisNode.IsActive = true;
thisNode.LastSeenDateTime = pollingTime;
thisNode.StoppedDateTime = null;
foreach (var otherNode in otherNodes)
{
otherNode.IsLeader = false;
}
rockContext.SaveChanges();
}
// Get polling wait time
var pollingWaitTimeSeconds = GetMaxPollingWaitSeconds();
// Wait a maximum of 10 seconds for responses
await Task.Delay(TimeSpan.FromSeconds(pollingWaitTimeSeconds)).ContinueWith(t =>
{
// Clear the ping pong key because responses are now late and no longer accepted
_leadershipPingKey = null;
Debug("Checking for unresponsive nodes");
using (var rockContext = new RockContext())
{
var webFarmNodeService = new WebFarmNodeService(rockContext);
var unresponsiveNodes = webFarmNodeService.Queryable()
.Where(wfn =>
wfn.LastSeenDateTime < pollingTime &&
wfn.IsActive &&
wfn.NodeName != NodeName)
.ToList();
Debug($"I found {unresponsiveNodes.Count} unresponsive nodes");
foreach (var node in unresponsiveNodes)
{
// Write to log if the server was thought to be active but did not respond
AddLog(rockContext, WebFarmNodeLog.SeverityLevel.Critical, node.Id, EventType.Availability, $"{node.NodeName} was marked active but did not respond to a ping");
node.IsActive = false;
}
rockContext.SaveChanges();
}
});
}
/// <summary>
/// Stage 1 Start.
/// Called before Hot load caches and start-up activities( EF migrations etc )
/// </summary>
public static void StartStage1()
{
if (_startStage != 0)
{
LogException($"Web Farm cannot start stage 1 when at stage {_startStage}");
return;
}
Debug("Start Stage 1");
using (var rockContext = new RockContext())
{
// Check that the WebFarmEnable = true.If yes, continue
if (!IsEnabled())
{
return;
}
// Check license key in the SystemSetting
if (!HasValidKey())
{
return;
}
_isWebFarmEnabledAndUnlocked = true;
// Initialize the performance counters that will be used when logging metrics
InitializePerformanceCounters();
// Load upper and lower polling interval settings
var lowerLimitSeconds = GetLowerPollingLimitSeconds();
var upperLimitSeconds = GetUpperPollingLimitSeconds();
// Find node record in DB using node name, if not found create a new record
var webFarmNodeService = new WebFarmNodeService(rockContext);
var webFarmNode = webFarmNodeService.Queryable().FirstOrDefault(wfn => wfn.NodeName == NodeName);
var isNewNode = webFarmNode == null;
if (isNewNode)
{
webFarmNode = new WebFarmNode
{
NodeName = NodeName
};
webFarmNodeService.Add(webFarmNode);
rockContext.SaveChanges();
}
_nodeId = webFarmNode.Id;
// Determine leadership polling interval. If provided in database( ConfiguredLeadershipPollingIntervalSeconds ) use that
// otherwise randomly select a number between upper and lower limits
const int maxGenerationAttempts = 100;
var generationAttempts = 1;
_pollingIntervalSeconds =
webFarmNode.ConfiguredLeadershipPollingIntervalSeconds ??
GeneratePollingIntervalSeconds(lowerLimitSeconds, upperLimitSeconds);
var isPollingIntervalInUse = IsPollingIntervalInUse(rockContext, NodeName, _pollingIntervalSeconds);
while (generationAttempts < maxGenerationAttempts && isPollingIntervalInUse)
{
generationAttempts++;
_pollingIntervalSeconds = GeneratePollingIntervalSeconds(lowerLimitSeconds, upperLimitSeconds);
isPollingIntervalInUse = IsPollingIntervalInUse(rockContext, NodeName, _pollingIntervalSeconds);
}
if (isPollingIntervalInUse)
{
var errorMessage = $"Web farm node {NodeName} did not successfully pick a polling interval after {maxGenerationAttempts} attempts";
AddLog(rockContext, WebFarmNodeLog.SeverityLevel.Warning, webFarmNode.Id, EventType.Error, errorMessage);
// Try to use the maximum value
_pollingIntervalSeconds = upperLimitSeconds;
isPollingIntervalInUse = IsPollingIntervalInUse(rockContext, NodeName, _pollingIntervalSeconds);
if (isPollingIntervalInUse)
{
LogException($"{errorMessage} and could not use the maximum polling limit");
return;
}
}
// Save the polling interval
// If web.config set to run jobs make IsCurrentJobRunner = true
// Set StoppedDateTime to null
// Update LastRestartDateTime to now
webFarmNode.CurrentLeadershipPollingIntervalSeconds = _pollingIntervalSeconds;
webFarmNode.IsCurrentJobRunner = IsCurrentJobRunner();
webFarmNode.StoppedDateTime = null;
webFarmNode.LastRestartDateTime = ProcessStartDateTime;
webFarmNode.LastSeenDateTime = ProcessStartDateTime;
webFarmNode.IsActive = false;
// Write to ClusterNodeLog -Startup Message
AddLog(rockContext, WebFarmNodeLog.SeverityLevel.Info, webFarmNode.Id, EventType.Startup, $"Process ID: {ProcessId}");
rockContext.SaveChanges();
}
_startStage = 1;
Debug("Done with Stage 1");
}
/// <summary>
/// Shows the mode where the user is only viewing an existing streak type
/// </summary>
private void ShowViewMode()
{
if (!IsViewMode())
{
return;
}
var canEdit = CanEdit();
btnEdit.Visible = canEdit;
pnlEditDetails.Visible = false;
pnlViewDetails.Visible = true;
HideSecondaryBlocks(false);
// Load values from system settings
var minPolling = RockWebFarm.GetLowerPollingLimitSeconds();
var maxPolling = RockWebFarm.GetUpperPollingLimitSeconds();
var minDifference = RockWebFarm.GetMinimumPollingDifferenceSeconds();
var pollingWait = RockWebFarm.GetMaxPollingWaitSeconds();
var maskedKey = SystemSettings.GetValue(SystemSetting.WEBFARM_KEY).Masked();
if (maskedKey.IsNullOrWhiteSpace())
{
maskedKey = "None";
}
// Build the description list with the values
var descriptionList = new DescriptionList();
descriptionList.Add("Key", string.Format("{0}", maskedKey));
descriptionList.Add("Min Polling Limit", string.Format("{0} seconds", minPolling));
descriptionList.Add("Max Polling Limit", string.Format("{0} seconds", maxPolling));
descriptionList.Add("Min Polling Difference", string.Format("{0} seconds", minDifference));
descriptionList.Add("Max Polling Wait", string.Format("{0} seconds", pollingWait));
var unresponsiveMinutes = 10;
var unresponsiveDateTime = RockDateTime.Now.AddMinutes(0 - unresponsiveMinutes);
// Bind the grid data view models
using (var rockContext = new RockContext())
{
var webFarmNodeService = new WebFarmNodeService(rockContext);
var webFarmNodeMetricService = new WebFarmNodeMetricService(rockContext);
var viewModels = webFarmNodeService.Queryable()
.AsNoTracking()
.Select(wfn => new WebFarmNodeService.NodeViewModel
{
PollingIntervalSeconds = wfn.CurrentLeadershipPollingIntervalSeconds,
IsJobRunner = wfn.IsCurrentJobRunner,
IsActive = wfn.IsActive,
IsUnresponsive = wfn.IsActive && !wfn.StoppedDateTime.HasValue && wfn.LastSeenDateTime < unresponsiveDateTime,
IsLeader = wfn.IsLeader,
NodeName = wfn.NodeName,
LastSeen = wfn.LastSeenDateTime,
Id = wfn.Id,
Metrics = wfn.WebFarmNodeMetrics
.Where(wfnm =>
wfnm.MetricType == WebFarmNodeMetric.TypeOfMetric.CpuUsagePercent &&
wfnm.MetricValueDateTime >= ChartMinDate &&
wfnm.MetricValueDateTime <= _chartMaxDate)
.Select(wfnm => new WebFarmNodeMetricService.MetricViewModel
{
MetricValueDateTime = wfnm.MetricValueDateTime,
MetricValue = wfnm.MetricValue
})
.ToList()
})
.ToList();
rNodes.DataSource = viewModels.OrderBy(n => n.NodeName);
rNodes.DataBind();
}
lDescription.Text = descriptionList.Html;
}
