private void bw_DoWork(object sender, DoWorkEventArgs e)
{
var worker = sender as BackgroundWorker;
var hruCol = new HealthRollupCollection(() => new HealthRollup());
var servers = ServerConfigMgr.GetServers();
int index = 0;
int iServerCount = servers.Count();
var checkedGroupServices = new ConcurrentBag <Tuple <ServerRole, HCWinService> >();
try
{
//Perform health check on each server
Parallel.ForEach(servers.ToList(), (server) =>
{
if (worker.CancellationPending || e.Cancel)
{
writeVerbose("Background worker is cancelling.");
e.Result = hruCol;
e.Cancel = true;
worker.ReportProgress(0, hruCol);
return;
}
server.IsOnline = GenericChecks.getIsOnline(server);
GenericChecks genChecks = new GenericChecks();
var winServicesToCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck()
.Where
(
x => x.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 ||
x.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 ||
!x.Function.Equals(ServerFunction.Unknown) && (x.Function.Equals(server.HostFunction) && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) ||
((x.Roles.Count + x.Servers.Count == 0) && (x.Function.Equals(ServerFunction.Unknown)))
).ToArray();
var tskGeneric = genChecks.PerformServerCheck(server);
var tskWinServices = GenericChecks.CheckWindowsServices(server, winServicesToCheck);
var tskListGrpWinServices = new List <Task>();
if (server.IsOnline && server.IsActive)
{
//For the services that have onepergroup set, calculate the servers that belong to the group
foreach (var grpWinService in winServicesToCheck.Where(x => x.OnePerGroup))
{
writeVerbose(string.Format("Checking group service {0} on {1}.", grpWinService.Name, server.HostName));
//Skip if we already have already calculated which servers belong to the group for this service
if (checkedGroupServices.Where(x => x.Item1 == server.HostRole).Select(x => x.Item2).Contains(grpWinService))
{
writeVerbose(string.Format("Already added group for service {0} for cluster containing {1}.", grpWinService.Name, server.HostName));
continue;
}
List <FiOSServer> grpServers = new List <FiOSServer>();
//if there are roles, add the groups of servers with the same roles, functions, locations, and servers that start with the same first 4 letters.
if (grpWinService.Roles.Count > 0)
{
writeVerbose(string.Format("First grpWinService.Roles.Count > 0 true for {0} on {1}", grpWinService.Name, server.HostName));
grpWinService.Roles.Where(x => x.Item2 == server.HostRole).ToList().ForEach((role) =>
{
grpServers.AddRange(servers.Where(x => (x.HostRole == role.Item2 && x.HostFunction == role.Item3) && role.Item1 && x.HostLocationName == server.HostLocationName &&
x.HostName.StartsWith(server.HostName.Substring(0, 4))).ToList());
});
}
//get server groups by host function and host name
else
{
writeVerbose(string.Format("First grpWinService.Roles.Count > 0 false for {0} on {1}", grpWinService.Name, server.HostName));
grpServers.AddRange(servers.Where(x => x.HostFunction == grpWinService.Function && x.HostName.StartsWith(server.HostName.Substring(0, 4))).ToList());
}
//Add directly included servers
if (grpWinService.Servers.Count > 0)
{
grpWinService.Servers.ForEach((svr) =>
{
grpServers.AddRange(servers.Where(x => (x.HostFullName == svr.Item2.HostFullName) && svr.Item1));
});
}
writeVerbose(string.Format("grpServers count = {0} for {1} on {2}", grpServers.Where(x => x.IsActive && x.IsOnline).Count(), grpWinService.Name, server.HostName));
//Do not check service if the server's name does not start with any of the group server's names
if (grpServers.Count > 0 && !grpServers.Any(x => x.HostName.ToUpper().StartsWith(server.HostName.ToUpper().Substring(0, 4))))
{
writeVerbose(string.Format("{0} is not a member of the server group for {1}.", server, grpWinService.Name));
continue;
}
//If there are multiple roles, add a task for each one so that they don't all group up under the same role in the results
if (grpWinService.Roles.Count > 1)
{
grpWinService.Roles.ForEach((role) =>
{
tskListGrpWinServices.Add(GenericChecks.CheckWindowsServices(grpServers.Where(x => x.HostRole == role.Item2 && x.IsActive && x.IsOnline).Select(x => x.HostName).ToArray(), grpWinService));
});
}
else
{
tskListGrpWinServices.Add(GenericChecks.CheckWindowsServices(grpServers.Where(x => x.IsActive && x.IsOnline).Select(x => x.HostFullName).ToArray(), grpWinService));
}
//Add to list to avoid grouping servers for the same service
checkedGroupServices.Add(new Tuple <ServerRole, HCWinService>(server.HostRole, grpWinService));
}
}
//Run general server checks
try
{
writeVerbose(string.Format("Beginning general server check task for {0}.", server.HostName));
tskGeneric.Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing general server checks on {0}. {1}", server.HostFullName, ex.Message), 10801, System.Diagnostics.TraceEventType.Error, true);
}
}
//Run group windows service checks
if (tskListGrpWinServices.Count > 0)
{
try
{
writeVerbose(string.Format("Beginning group tasks for {0}.", server.HostName));
Task.WaitAll(tskListGrpWinServices.ToArray());
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing windows service check on server group containing {0}", server.HostFullName), 10801, System.Diagnostics.TraceEventType.Error, true);
}
}
}
//Run windows service checks
try
{
writeVerbose(string.Format("Beginning windows service check task for {0}.", server.HostName));
tskWinServices.Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing windows service check on {0}. {1}", server.HostFullName, ex.Message), 10802, System.Diagnostics.TraceEventType.Error, true);
}
}
//Get results for all tasks
try
{
if (tskGeneric.Status == TaskStatus.RanToCompletion)
{
hruCol.PutObject(tskGeneric.Result);
}
writeVerbose(string.Format("Finished general server check task for {0}.", server.HostName));
if (tskWinServices.Status == TaskStatus.RanToCompletion)
{
foreach (var result in tskWinServices.Result)
{
hruCol.PutObject(result);
}
}
writeVerbose(string.Format("Finished windows service check task for {0}.", server.HostName));
//Group services results
foreach (Task <List <HealthCheckError> > tsk in tskListGrpWinServices)
{
if (tsk.Status == TaskStatus.RanToCompletion)
{
hruCol.PutObject(new HealthRollup()
{
Server = server, Errors = tsk.Result
});
}
}
writeVerbose(string.Format("Finished group tasks for {0}.", server.HostName));
}
catch (Exception ex)
{
throw new Exception(string.Format("Failed to add health check for {0} to collection. {1}", server.HostName, ex.Message));
}
//Web server checks
if (server is FiOSWebServer)
{
writeVerbose(string.Format("Beginning web server checks task for {0}.", server.HostName));
IISChecks iisChecks = new IISChecks();
var tskIISChecks = iisChecks.CheckWebServer(server as FiOSWebServer);
try
{
tskIISChecks.Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing IIS checks on {0}. {1}.", server.HostFullName, ex.Message), 10801, System.Diagnostics.TraceEventType.Error, true);
}
}
try
{
if (tskIISChecks.Status == TaskStatus.RanToCompletion)
{
hruCol.PutObject(tskIISChecks.Result);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Failed to add web health check for {0} to collection. {1}", server.HostName, ex.Message));
}
writeVerbose(string.Format("Finished web server checks task for {0}.", server.HostName));
}
bw.ReportProgress((int)(((decimal)++ index / (decimal)iServerCount) * 100), hruCol);
writeVerbose(string.Format("Finished reporting progress for {0}. {1} / {2}", server.HostName, index, iServerCount));
});
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
System.Diagnostics.Debug.WriteLine(ex.StackTrace);
writeError(string.Format("Error in server checks. {0}", ex.Message), 10805, System.Diagnostics.TraceEventType.Error);
}
}
catch (Exception ex)
{
writeError(string.Format("Error while performing checks.", ex.Message), 10800, System.Diagnostics.TraceEventType.Error);
}
e.Result = hruCol;
}