private void bw_DoWork(object sender, DoWorkEventArgs e)
{
var worker = sender as BackgroundWorker;
var hruCol = new HealthRollupCollection(() => new HealthRollup());
var servers = ServerConfigMgr.GetServers();
int index = 0;
int iServerCount = servers.Count();
var checkedGroupServices = new ConcurrentBag <Tuple <ServerRole, HCWinService> >();
try
{
//Perform health check on each server
Parallel.ForEach(servers.ToList(), (server) =>
{
if (worker.CancellationPending || e.Cancel)
{
writeVerbose("Background worker is cancelling.");
e.Result = hruCol;
e.Cancel = true;
worker.ReportProgress(0, hruCol);
return;
}
server.IsOnline = GenericChecks.getIsOnline(server);
GenericChecks genChecks = new GenericChecks();
var winServicesToCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck()
.Where
(
x => x.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 ||
x.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 ||
!x.Function.Equals(ServerFunction.Unknown) && (x.Function.Equals(server.HostFunction) && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) ||
((x.Roles.Count + x.Servers.Count == 0) && (x.Function.Equals(ServerFunction.Unknown)))
).ToArray();
var tskGeneric = genChecks.PerformServerCheck(server);
var tskWinServices = GenericChecks.CheckWindowsServices(server, winServicesToCheck);
var tskListGrpWinServices = new List <Task>();
if (server.IsOnline && server.IsActive)
{
//For the services that have onepergroup set, calculate the servers that belong to the group
foreach (var grpWinService in winServicesToCheck.Where(x => x.OnePerGroup))
{
writeVerbose(string.Format("Checking group service {0} on {1}.", grpWinService.Name, server.HostName));
//Skip if we already have already calculated which servers belong to the group for this service
if (checkedGroupServices.Where(x => x.Item1 == server.HostRole).Select(x => x.Item2).Contains(grpWinService))
{
writeVerbose(string.Format("Already added group for service {0} for cluster containing {1}.", grpWinService.Name, server.HostName));
continue;
}
List <FiOSServer> grpServers = new List <FiOSServer>();
//if there are roles, add the groups of servers with the same roles, functions, locations, and servers that start with the same first 4 letters.
if (grpWinService.Roles.Count > 0)
{
writeVerbose(string.Format("First grpWinService.Roles.Count > 0 true for {0} on {1}", grpWinService.Name, server.HostName));
grpWinService.Roles.Where(x => x.Item2 == server.HostRole).ToList().ForEach((role) =>
{
grpServers.AddRange(servers.Where(x => (x.HostRole == role.Item2 && x.HostFunction == role.Item3) && role.Item1 && x.HostLocationName == server.HostLocationName &&
x.HostName.StartsWith(server.HostName.Substring(0, 4))).ToList());
});
}
//get server groups by host function and host name
else
{
writeVerbose(string.Format("First grpWinService.Roles.Count > 0 false for {0} on {1}", grpWinService.Name, server.HostName));
grpServers.AddRange(servers.Where(x => x.HostFunction == grpWinService.Function && x.HostName.StartsWith(server.HostName.Substring(0, 4))).ToList());
}
//Add directly included servers
if (grpWinService.Servers.Count > 0)
{
grpWinService.Servers.ForEach((svr) =>
{
grpServers.AddRange(servers.Where(x => (x.HostFullName == svr.Item2.HostFullName) && svr.Item1));
});
}
writeVerbose(string.Format("grpServers count = {0} for {1} on {2}", grpServers.Where(x => x.IsActive && x.IsOnline).Count(), grpWinService.Name, server.HostName));
//Do not check service if the server's name does not start with any of the group server's names
if (grpServers.Count > 0 && !grpServers.Any(x => x.HostName.ToUpper().StartsWith(server.HostName.ToUpper().Substring(0, 4))))
{
writeVerbose(string.Format("{0} is not a member of the server group for {1}.", server, grpWinService.Name));
continue;
}
//If there are multiple roles, add a task for each one so that they don't all group up under the same role in the results
if (grpWinService.Roles.Count > 1)
{
grpWinService.Roles.ForEach((role) =>
{
tskListGrpWinServices.Add(GenericChecks.CheckWindowsServices(grpServers.Where(x => x.HostRole == role.Item2 && x.IsActive && x.IsOnline).Select(x => x.HostName).ToArray(), grpWinService));
});
}
else
{
tskListGrpWinServices.Add(GenericChecks.CheckWindowsServices(grpServers.Where(x => x.IsActive && x.IsOnline).Select(x => x.HostFullName).ToArray(), grpWinService));
}
//Add to list to avoid grouping servers for the same service
checkedGroupServices.Add(new Tuple <ServerRole, HCWinService>(server.HostRole, grpWinService));
}
}
//Run general server checks
try
{
writeVerbose(string.Format("Beginning general server check task for {0}.", server.HostName));
tskGeneric.Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing general server checks on {0}. {1}", server.HostFullName, ex.Message), 10801, System.Diagnostics.TraceEventType.Error, true);
}
}
//Run group windows service checks
if (tskListGrpWinServices.Count > 0)
{
try
{
writeVerbose(string.Format("Beginning group tasks for {0}.", server.HostName));
Task.WaitAll(tskListGrpWinServices.ToArray());
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing windows service check on server group containing {0}", server.HostFullName), 10801, System.Diagnostics.TraceEventType.Error, true);
}
}
}
//Run windows service checks
try
{
writeVerbose(string.Format("Beginning windows service check task for {0}.", server.HostName));
tskWinServices.Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing windows service check on {0}. {1}", server.HostFullName, ex.Message), 10802, System.Diagnostics.TraceEventType.Error, true);
}
}
//Get results for all tasks
try
{
if (tskGeneric.Status == TaskStatus.RanToCompletion)
{
hruCol.PutObject(tskGeneric.Result);
}
writeVerbose(string.Format("Finished general server check task for {0}.", server.HostName));
if (tskWinServices.Status == TaskStatus.RanToCompletion)
{
foreach (var result in tskWinServices.Result)
{
hruCol.PutObject(result);
}
}
writeVerbose(string.Format("Finished windows service check task for {0}.", server.HostName));
//Group services results
foreach (Task <List <HealthCheckError> > tsk in tskListGrpWinServices)
{
if (tsk.Status == TaskStatus.RanToCompletion)
{
hruCol.PutObject(new HealthRollup()
{
Server = server, Errors = tsk.Result
});
}
}
writeVerbose(string.Format("Finished group tasks for {0}.", server.HostName));
}
catch (Exception ex)
{
throw new Exception(string.Format("Failed to add health check for {0} to collection. {1}", server.HostName, ex.Message));
}
//Web server checks
if (server is FiOSWebServer)
{
writeVerbose(string.Format("Beginning web server checks task for {0}.", server.HostName));
IISChecks iisChecks = new IISChecks();
var tskIISChecks = iisChecks.CheckWebServer(server as FiOSWebServer);
try
{
tskIISChecks.Wait();
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
writeEvent(string.Format("Error while performing IIS checks on {0}. {1}.", server.HostFullName, ex.Message), 10801, System.Diagnostics.TraceEventType.Error, true);
}
}
try
{
if (tskIISChecks.Status == TaskStatus.RanToCompletion)
{
hruCol.PutObject(tskIISChecks.Result);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Failed to add web health check for {0} to collection. {1}", server.HostName, ex.Message));
}
writeVerbose(string.Format("Finished web server checks task for {0}.", server.HostName));
}
bw.ReportProgress((int)(((decimal)++ index / (decimal)iServerCount) * 100), hruCol);
writeVerbose(string.Format("Finished reporting progress for {0}. {1} / {2}", server.HostName, index, iServerCount));
});
}
catch (AggregateException aex)
{
foreach (var ex in aex.Flatten().InnerExceptions)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
System.Diagnostics.Debug.WriteLine(ex.StackTrace);
writeError(string.Format("Error in server checks. {0}", ex.Message), 10805, System.Diagnostics.TraceEventType.Error);
}
}
catch (Exception ex)
{
writeError(string.Format("Error while performing checks.", ex.Message), 10800, System.Diagnostics.TraceEventType.Error);
}
e.Result = hruCol;
}
static void Main(string[] args)
{
var server = ServerConfigMgr.GetServers().Where(x => x.HostName.ToUpper().Equals("CTXV01PIMGD01")).FirstOrDefault();
bool isExempt = ServerHealthCheckConfigMgr.IsExempt(server, ExemptionType.HardDrive, "O");
var winServicesToCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck()
.Where
(
x => x.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 ||
x.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 ||
!x.Function.Equals(ServerFunction.Unknown) && (x.Function.Equals(server.HostFunction) && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) ||
((x.Roles.Count + x.Servers.Count == 0) && (x.Function.Equals(ServerFunction.Unknown)))
).ToArray();
//hcWServices = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck().Where(x => x.OnePerGroup).ToArray();
foreach (var svc in ServerHealthCheckConfigMgr.GetWindowsServicesToCheck().Where(x => x.Name == "WAS"))
{
var count = svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count();
var count2 = svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count();
//if (svc.Roles.Count >= 1)
//svc.Roles.ForEach(x => Console.WriteLine("{3} - {0} | {1} | {2}", x.Item1, x.Item2, x.Item3, svc.Name));
bool bol1 = svc.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0;
bool bol2 = svc.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0;
bool bol3 = (svc.Function.Equals(server.HostFunction) && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0);
bool bol4 = ((svc.Roles.Count + svc.Servers.Count == 0) && (svc.Function.Equals(ServerFunction.Unknown)));
bool bol5 = svc.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 ||
svc.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 ||
(svc.Function.Equals(server.HostFunction) && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) ||
((svc.Roles.Count + svc.Servers.Count == 0) && (svc.Function.Equals(ServerFunction.Unknown)));
Console.WriteLine("{0} | {1} | {2} | {3} | {4} | {5} | {6} | {7} | {8}", svc.Name, count, server.HostFunction, server.HostRole, count2, bol1, bol2, bol3, bol4, bol5);
}
//var roleCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck().Where(x => x.Roles.Where(y => y.Item1 && y.Item2 == svr.HostRole).Count() > 0);
foreach (var service in winServicesToCheck)
{
Console.WriteLine("Name:\t\t{0}", service.Name);
Console.WriteLine("Display:\t{0}", service.DisplayName);
if (service.CheckStatus.Count > 0)
{
service.CheckStatus.ForEach(x => Console.WriteLine("Status:\t\t{0}", x.ToString()));
}
if (service.CheckStartupType.Count > 0)
{
service.CheckStartupType.ForEach(x => Console.WriteLine("StartupType:\t{0}", x.ToString()));
}
if (service.CheckLogonAs.Count > 0)
{
service.CheckLogonAs.ForEach(x => Console.WriteLine("LogonAs:\t{0} - {1}", x, GenericChecks.GetWindowsServiceLogonAccount("B1ZHN32", service.Name)));
}
if (service.Roles.Count > 0)
{
service.Roles.ForEach((x) =>
{
if (x.Item1)
{
Console.WriteLine("INCLUDE ROLES");
}
else
{
Console.WriteLine("EXCLUDE ROLES");
}
Console.WriteLine("Role:\t\t{0}", x.Item2.ToString());
Console.WriteLine("Function:\t{0}", x.Item3.ToString());
});
}
if (service.Servers.Count > 0)
{
service.Servers.ForEach((x) =>
{
if (x.Item1)
{
Console.WriteLine("INCLUDE SERVERS");
}
else
{
Console.WriteLine("EXCLUDE SERVERS");
}
Console.WriteLine("Server Name:\t{0}", x.Item2.HostName);
Console.WriteLine("Server IP:\t{0}", x.Item2.IPAddress);
Console.WriteLine("Server Location:\t{0}", x.Item2.HostLocation.ToString());
});
}
Console.WriteLine("OnePerGroup:\t{0}", service.OnePerGroup);
Console.WriteLine();
}
Console.WriteLine("Press Enter to Continue...");
Console.ReadLine();
HTMLFormatter formatter = new HTMLFormatter();
formatter.SetRole(ServerRole.NSP.ToString());
formatter.BeginTable("NSPTXCAWAPV01");
formatter.AddStatusRow("Server Status", StatusResult.Ok);
formatter.AddStatusRow("Web Services", StatusResult.Error);
List <string> errors = new List <string>();
errors.Add("Config file mismatch with NSPTXCAWAPV02");
errors.Add("Web API returned error");
formatter.AddErrorDescriptionRows(errors);
formatter.AddStatusRow("Database", StatusResult.Error);
errors.Clear();
errors.Add("Missing data in table");
formatter.AddErrorDescriptionRows(errors);
formatter.EndTable();
formatter.BeginTable("NSPTXCAWAPV02");
formatter.AddStatusRow("Server Status", StatusResult.Ok);
formatter.AddStatusRow("Web Services", StatusResult.Error);
errors.Clear();
errors.Add("Config file mismatch with NSPTXCAWAPV01");
formatter.AddErrorDescriptionRows(errors);
formatter.AddStatusRow("Database", StatusResult.Ok);
formatter.EndTable();
formatter.BeginTable("NSPTXCAWAPV03");
formatter.AddStatusRow("Server Status", StatusResult.Critical);
errors.Clear();
errors.Add("Server is unreachable.");
formatter.AddErrorDescriptionRows(errors);
formatter.AddStatusRow("Web Services", StatusResult.Ok);
formatter.AddStatusRow("Database", StatusResult.Ok);
formatter.EndTable();
Toolset.SendEmail("mailrelay.corp.pvt", null, 25, false, "HealthCheck Test Email", formatter.ToString(), "*****@*****.**", new string[1] {
"*****@*****.**"
}, null);
var servers = ServerConfigMgr.GetServers().ToList();
foreach (var svr in servers.Where(x => x.HostRole == ServerRole.IMG))
{
if (svr is FiOSDbServer)
{
Console.WriteLine("\nDATABASE:\n");
Console.WriteLine((svr as FiOSDbServer).DatabaseType);
}
else if (svr is FiOSWebServer)
{
Console.WriteLine("\nIIS SERVER:\n");
}
else
{
Console.Write("\nFiOS Server:\n");
}
Console.WriteLine(svr.HostName);
Console.WriteLine(svr.HostFullName);
Console.WriteLine(svr.HostFunction);
Console.WriteLine(svr.HostLocation);
Console.WriteLine(svr.HostLocationName);
Console.WriteLine(svr.HostName);
Console.WriteLine(svr.HostRole);
Console.WriteLine(svr.IsOnline);
try
{
if (svr is FiOSWebServer)
{
using (var iisMgr = new IISServerMgr(svr))
{
try
{
var Sites = iisMgr.GetSiteCollection();
foreach (var site in Sites)
{
Console.WriteLine("Site: {0} - {1} - {2}", site.Name, site.ServerAutoStart, site.State);
foreach (var app in site.Applications)
{
Console.WriteLine("App Pool: {0} - {1}", app.ApplicationPoolName, app.Path);
foreach (var vd in app.VirtualDirectories)
{
Console.WriteLine("Virtual Dir: {0} - {1}", vd.Path, vd.PhysicalPath);
foreach (var attr in vd.Attributes)
{
Console.WriteLine("Attribute: {0} - {1}", attr.Name, attr.Value);
}
foreach (var ce in vd.ChildElements)
{
foreach (var attr in ce.Attributes)
{
Console.WriteLine("Child Attribute: {0} - {1}", attr.Name, attr.Value);
}
}
}
}
}
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed to get Sites for {0}. {1}", svr.HostName, ex.Message);
Console.ResetColor();
}
}
}
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("General Error: {0}", ex.Message);
Console.ResetColor();
}
}
Console.WriteLine("\n\nPress any key to exit...");
Console.ReadLine();
}