private void bw_DoWork(object sender, DoWorkEventArgs e) { var worker = sender as BackgroundWorker; var hruCol = new HealthRollupCollection(() => new HealthRollup()); var servers = ServerConfigMgr.GetServers(); int index = 0; int iServerCount = servers.Count(); var checkedGroupServices = new ConcurrentBag <Tuple <ServerRole, HCWinService> >(); try { //Perform health check on each server Parallel.ForEach(servers.ToList(), (server) => { if (worker.CancellationPending || e.Cancel) { writeVerbose("Background worker is cancelling."); e.Result = hruCol; e.Cancel = true; worker.ReportProgress(0, hruCol); return; } server.IsOnline = GenericChecks.getIsOnline(server); GenericChecks genChecks = new GenericChecks(); var winServicesToCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck() .Where ( x => x.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 || x.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 || !x.Function.Equals(ServerFunction.Unknown) && (x.Function.Equals(server.HostFunction) && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) || ((x.Roles.Count + x.Servers.Count == 0) && (x.Function.Equals(ServerFunction.Unknown))) ).ToArray(); var tskGeneric = genChecks.PerformServerCheck(server); var tskWinServices = GenericChecks.CheckWindowsServices(server, winServicesToCheck); var tskListGrpWinServices = new List <Task>(); if (server.IsOnline && server.IsActive) { //For the services that have onepergroup set, calculate the servers that belong to the group foreach (var grpWinService in winServicesToCheck.Where(x => x.OnePerGroup)) { writeVerbose(string.Format("Checking group service {0} on {1}.", grpWinService.Name, server.HostName)); //Skip if we already have already calculated which servers belong to the group for this service if (checkedGroupServices.Where(x => x.Item1 == server.HostRole).Select(x => x.Item2).Contains(grpWinService)) { writeVerbose(string.Format("Already added group for service {0} for cluster containing {1}.", grpWinService.Name, server.HostName)); continue; } List <FiOSServer> grpServers = new List <FiOSServer>(); //if there are roles, add the groups of servers with the same roles, functions, locations, and servers that start with the same first 4 letters. if (grpWinService.Roles.Count > 0) { writeVerbose(string.Format("First grpWinService.Roles.Count > 0 true for {0} on {1}", grpWinService.Name, server.HostName)); grpWinService.Roles.Where(x => x.Item2 == server.HostRole).ToList().ForEach((role) => { grpServers.AddRange(servers.Where(x => (x.HostRole == role.Item2 && x.HostFunction == role.Item3) && role.Item1 && x.HostLocationName == server.HostLocationName && x.HostName.StartsWith(server.HostName.Substring(0, 4))).ToList()); }); } //get server groups by host function and host name else { writeVerbose(string.Format("First grpWinService.Roles.Count > 0 false for {0} on {1}", grpWinService.Name, server.HostName)); grpServers.AddRange(servers.Where(x => x.HostFunction == grpWinService.Function && x.HostName.StartsWith(server.HostName.Substring(0, 4))).ToList()); } //Add directly included servers if (grpWinService.Servers.Count > 0) { grpWinService.Servers.ForEach((svr) => { grpServers.AddRange(servers.Where(x => (x.HostFullName == svr.Item2.HostFullName) && svr.Item1)); }); } writeVerbose(string.Format("grpServers count = {0} for {1} on {2}", grpServers.Where(x => x.IsActive && x.IsOnline).Count(), grpWinService.Name, server.HostName)); //Do not check service if the server's name does not start with any of the group server's names if (grpServers.Count > 0 && !grpServers.Any(x => x.HostName.ToUpper().StartsWith(server.HostName.ToUpper().Substring(0, 4)))) { writeVerbose(string.Format("{0} is not a member of the server group for {1}.", server, grpWinService.Name)); continue; } //If there are multiple roles, add a task for each one so that they don't all group up under the same role in the results if (grpWinService.Roles.Count > 1) { grpWinService.Roles.ForEach((role) => { tskListGrpWinServices.Add(GenericChecks.CheckWindowsServices(grpServers.Where(x => x.HostRole == role.Item2 && x.IsActive && x.IsOnline).Select(x => x.HostName).ToArray(), grpWinService)); }); } else { tskListGrpWinServices.Add(GenericChecks.CheckWindowsServices(grpServers.Where(x => x.IsActive && x.IsOnline).Select(x => x.HostFullName).ToArray(), grpWinService)); } //Add to list to avoid grouping servers for the same service checkedGroupServices.Add(new Tuple <ServerRole, HCWinService>(server.HostRole, grpWinService)); } } //Run general server checks try { writeVerbose(string.Format("Beginning general server check task for {0}.", server.HostName)); tskGeneric.Wait(); } catch (AggregateException aex) { foreach (var ex in aex.Flatten().InnerExceptions) { writeEvent(string.Format("Error while performing general server checks on {0}. {1}", server.HostFullName, ex.Message), 10801, System.Diagnostics.TraceEventType.Error, true); } } //Run group windows service checks if (tskListGrpWinServices.Count > 0) { try { writeVerbose(string.Format("Beginning group tasks for {0}.", server.HostName)); Task.WaitAll(tskListGrpWinServices.ToArray()); } catch (AggregateException aex) { foreach (var ex in aex.Flatten().InnerExceptions) { writeEvent(string.Format("Error while performing windows service check on server group containing {0}", server.HostFullName), 10801, System.Diagnostics.TraceEventType.Error, true); } } } //Run windows service checks try { writeVerbose(string.Format("Beginning windows service check task for {0}.", server.HostName)); tskWinServices.Wait(); } catch (AggregateException aex) { foreach (var ex in aex.Flatten().InnerExceptions) { writeEvent(string.Format("Error while performing windows service check on {0}. {1}", server.HostFullName, ex.Message), 10802, System.Diagnostics.TraceEventType.Error, true); } } //Get results for all tasks try { if (tskGeneric.Status == TaskStatus.RanToCompletion) { hruCol.PutObject(tskGeneric.Result); } writeVerbose(string.Format("Finished general server check task for {0}.", server.HostName)); if (tskWinServices.Status == TaskStatus.RanToCompletion) { foreach (var result in tskWinServices.Result) { hruCol.PutObject(result); } } writeVerbose(string.Format("Finished windows service check task for {0}.", server.HostName)); //Group services results foreach (Task <List <HealthCheckError> > tsk in tskListGrpWinServices) { if (tsk.Status == TaskStatus.RanToCompletion) { hruCol.PutObject(new HealthRollup() { Server = server, Errors = tsk.Result }); } } writeVerbose(string.Format("Finished group tasks for {0}.", server.HostName)); } catch (Exception ex) { throw new Exception(string.Format("Failed to add health check for {0} to collection. {1}", server.HostName, ex.Message)); } //Web server checks if (server is FiOSWebServer) { writeVerbose(string.Format("Beginning web server checks task for {0}.", server.HostName)); IISChecks iisChecks = new IISChecks(); var tskIISChecks = iisChecks.CheckWebServer(server as FiOSWebServer); try { tskIISChecks.Wait(); } catch (AggregateException aex) { foreach (var ex in aex.Flatten().InnerExceptions) { writeEvent(string.Format("Error while performing IIS checks on {0}. {1}.", server.HostFullName, ex.Message), 10801, System.Diagnostics.TraceEventType.Error, true); } } try { if (tskIISChecks.Status == TaskStatus.RanToCompletion) { hruCol.PutObject(tskIISChecks.Result); } } catch (Exception ex) { throw new Exception(string.Format("Failed to add web health check for {0} to collection. {1}", server.HostName, ex.Message)); } writeVerbose(string.Format("Finished web server checks task for {0}.", server.HostName)); } bw.ReportProgress((int)(((decimal)++ index / (decimal)iServerCount) * 100), hruCol); writeVerbose(string.Format("Finished reporting progress for {0}. {1} / {2}", server.HostName, index, iServerCount)); }); } catch (AggregateException aex) { foreach (var ex in aex.Flatten().InnerExceptions) { System.Diagnostics.Debug.WriteLine(ex.Message); System.Diagnostics.Debug.WriteLine(ex.StackTrace); writeError(string.Format("Error in server checks. {0}", ex.Message), 10805, System.Diagnostics.TraceEventType.Error); } } catch (Exception ex) { writeError(string.Format("Error while performing checks.", ex.Message), 10800, System.Diagnostics.TraceEventType.Error); } e.Result = hruCol; }
static void Main(string[] args) { var server = ServerConfigMgr.GetServers().Where(x => x.HostName.ToUpper().Equals("CTXV01PIMGD01")).FirstOrDefault(); bool isExempt = ServerHealthCheckConfigMgr.IsExempt(server, ExemptionType.HardDrive, "O"); var winServicesToCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck() .Where ( x => x.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 || x.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 || !x.Function.Equals(ServerFunction.Unknown) && (x.Function.Equals(server.HostFunction) && x.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + x.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) || ((x.Roles.Count + x.Servers.Count == 0) && (x.Function.Equals(ServerFunction.Unknown))) ).ToArray(); //hcWServices = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck().Where(x => x.OnePerGroup).ToArray(); foreach (var svc in ServerHealthCheckConfigMgr.GetWindowsServicesToCheck().Where(x => x.Name == "WAS")) { var count = svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count(); var count2 = svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count(); //if (svc.Roles.Count >= 1) //svc.Roles.ForEach(x => Console.WriteLine("{3} - {0} | {1} | {2}", x.Item1, x.Item2, x.Item3, svc.Name)); bool bol1 = svc.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0; bool bol2 = svc.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0; bool bol3 = (svc.Function.Equals(server.HostFunction) && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0); bool bol4 = ((svc.Roles.Count + svc.Servers.Count == 0) && (svc.Function.Equals(ServerFunction.Unknown))); bool bol5 = svc.Servers.Where(y => y.Item1 && y.Item2.HostName == server.HostName).Count() > 0 && svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0 || svc.Roles.Where(y => y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() > 0 && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() == 0 || (svc.Function.Equals(server.HostFunction) && svc.Servers.Where(y => !y.Item1 && y.Item2.HostName == server.HostName).Count() + svc.Roles.Where(y => !y.Item1 && y.Item2 == server.HostRole && y.Item3 == server.HostFunction).Count() == 0) || ((svc.Roles.Count + svc.Servers.Count == 0) && (svc.Function.Equals(ServerFunction.Unknown))); Console.WriteLine("{0} | {1} | {2} | {3} | {4} | {5} | {6} | {7} | {8}", svc.Name, count, server.HostFunction, server.HostRole, count2, bol1, bol2, bol3, bol4, bol5); } //var roleCheck = ServerHealthCheckConfigMgr.GetWindowsServicesToCheck().Where(x => x.Roles.Where(y => y.Item1 && y.Item2 == svr.HostRole).Count() > 0); foreach (var service in winServicesToCheck) { Console.WriteLine("Name:\t\t{0}", service.Name); Console.WriteLine("Display:\t{0}", service.DisplayName); if (service.CheckStatus.Count > 0) { service.CheckStatus.ForEach(x => Console.WriteLine("Status:\t\t{0}", x.ToString())); } if (service.CheckStartupType.Count > 0) { service.CheckStartupType.ForEach(x => Console.WriteLine("StartupType:\t{0}", x.ToString())); } if (service.CheckLogonAs.Count > 0) { service.CheckLogonAs.ForEach(x => Console.WriteLine("LogonAs:\t{0} - {1}", x, GenericChecks.GetWindowsServiceLogonAccount("B1ZHN32", service.Name))); } if (service.Roles.Count > 0) { service.Roles.ForEach((x) => { if (x.Item1) { Console.WriteLine("INCLUDE ROLES"); } else { Console.WriteLine("EXCLUDE ROLES"); } Console.WriteLine("Role:\t\t{0}", x.Item2.ToString()); Console.WriteLine("Function:\t{0}", x.Item3.ToString()); }); } if (service.Servers.Count > 0) { service.Servers.ForEach((x) => { if (x.Item1) { Console.WriteLine("INCLUDE SERVERS"); } else { Console.WriteLine("EXCLUDE SERVERS"); } Console.WriteLine("Server Name:\t{0}", x.Item2.HostName); Console.WriteLine("Server IP:\t{0}", x.Item2.IPAddress); Console.WriteLine("Server Location:\t{0}", x.Item2.HostLocation.ToString()); }); } Console.WriteLine("OnePerGroup:\t{0}", service.OnePerGroup); Console.WriteLine(); } Console.WriteLine("Press Enter to Continue..."); Console.ReadLine(); HTMLFormatter formatter = new HTMLFormatter(); formatter.SetRole(ServerRole.NSP.ToString()); formatter.BeginTable("NSPTXCAWAPV01"); formatter.AddStatusRow("Server Status", StatusResult.Ok); formatter.AddStatusRow("Web Services", StatusResult.Error); List <string> errors = new List <string>(); errors.Add("Config file mismatch with NSPTXCAWAPV02"); errors.Add("Web API returned error"); formatter.AddErrorDescriptionRows(errors); formatter.AddStatusRow("Database", StatusResult.Error); errors.Clear(); errors.Add("Missing data in table"); formatter.AddErrorDescriptionRows(errors); formatter.EndTable(); formatter.BeginTable("NSPTXCAWAPV02"); formatter.AddStatusRow("Server Status", StatusResult.Ok); formatter.AddStatusRow("Web Services", StatusResult.Error); errors.Clear(); errors.Add("Config file mismatch with NSPTXCAWAPV01"); formatter.AddErrorDescriptionRows(errors); formatter.AddStatusRow("Database", StatusResult.Ok); formatter.EndTable(); formatter.BeginTable("NSPTXCAWAPV03"); formatter.AddStatusRow("Server Status", StatusResult.Critical); errors.Clear(); errors.Add("Server is unreachable."); formatter.AddErrorDescriptionRows(errors); formatter.AddStatusRow("Web Services", StatusResult.Ok); formatter.AddStatusRow("Database", StatusResult.Ok); formatter.EndTable(); Toolset.SendEmail("mailrelay.corp.pvt", null, 25, false, "HealthCheck Test Email", formatter.ToString(), "*****@*****.**", new string[1] { "*****@*****.**" }, null); var servers = ServerConfigMgr.GetServers().ToList(); foreach (var svr in servers.Where(x => x.HostRole == ServerRole.IMG)) { if (svr is FiOSDbServer) { Console.WriteLine("\nDATABASE:\n"); Console.WriteLine((svr as FiOSDbServer).DatabaseType); } else if (svr is FiOSWebServer) { Console.WriteLine("\nIIS SERVER:\n"); } else { Console.Write("\nFiOS Server:\n"); } Console.WriteLine(svr.HostName); Console.WriteLine(svr.HostFullName); Console.WriteLine(svr.HostFunction); Console.WriteLine(svr.HostLocation); Console.WriteLine(svr.HostLocationName); Console.WriteLine(svr.HostName); Console.WriteLine(svr.HostRole); Console.WriteLine(svr.IsOnline); try { if (svr is FiOSWebServer) { using (var iisMgr = new IISServerMgr(svr)) { try { var Sites = iisMgr.GetSiteCollection(); foreach (var site in Sites) { Console.WriteLine("Site: {0} - {1} - {2}", site.Name, site.ServerAutoStart, site.State); foreach (var app in site.Applications) { Console.WriteLine("App Pool: {0} - {1}", app.ApplicationPoolName, app.Path); foreach (var vd in app.VirtualDirectories) { Console.WriteLine("Virtual Dir: {0} - {1}", vd.Path, vd.PhysicalPath); foreach (var attr in vd.Attributes) { Console.WriteLine("Attribute: {0} - {1}", attr.Name, attr.Value); } foreach (var ce in vd.ChildElements) { foreach (var attr in ce.Attributes) { Console.WriteLine("Child Attribute: {0} - {1}", attr.Name, attr.Value); } } } } } } catch (Exception ex) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Failed to get Sites for {0}. {1}", svr.HostName, ex.Message); Console.ResetColor(); } } } } catch (Exception ex) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("General Error: {0}", ex.Message); Console.ResetColor(); } } Console.WriteLine("\n\nPress any key to exit..."); Console.ReadLine(); }
/// <summary> /// Performs all server checks on the server /// </summary> /// <param name="Server">FiOS Server</param> /// <returns>A health rollup containing all general server checks</returns> public async Task <HealthRollup> PerformServerCheck(FiOSServer Server) { var hru = new HealthRollup(); var hce = new HealthCheckError(); hru.Server = Server; if (!Server.IsActive) { hce.Result = StatusResult.Skipped; hce.HCType = HealthCheckType.GeneralServer; if (Server.IsOnline) { hce.Error.Add("Server is marked as inactive but is currently online."); } else { hce.Error.Add("Server is offline"); } hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } if (!Server.IsOnline) { hce.Error.Add(string.Format("{0} - Server is offline or unreachable via it's FQDN.", StatusResult.Critical)); hce.Result = StatusResult.Critical; hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } #region CheckHardDrives try { var hdds = await HardDrive.GetHardDriveAsync(Server.HostFullName); for (int i = 0; i < hdds.Length; i++) { var result = StatusResult.Ok; //If server is exempt, skip if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.HardDrive, hdds[i].DriveLetter)) { hce.Error.Add(string.Format("Skipped hard drive checks for drive letter {0}. {1}GB Remaining.", hdds[i].DriveLetter, ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1"))); continue; } //Check Hard Drive Space try { result = await checkHDDSpaceAsync(hdds[i]); if (result != StatusResult.Ok) { hce.Error.Add(string.Format("{0} - Disk {1} currently only has {2}GB of {3}GB of space remaining. SN: {4}.", result, hdds[i].Name, ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1"), ((decimal)hdds[i].Capacity / 1024M / 1024M / 1024M).ToString("N1"), hdds[i].SerialNumber)); hce.Result = getCorrectStatusResult(hce.Result, result); } } catch (Exception ex) { hce.Error.Add(string.Format("{0} - Failed to get available disk space for {2}. Exception: {1}", StatusResult.Error, ex.Message, hdds[i].DriveLetter)); hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error); } //Check Hard Drive Status try { result = await checkHDDStatusAsync(hdds[i]); hce.Result = getCorrectStatusResult(hce.Result, result); if (result != StatusResult.Ok) { hce.Error.Add(string.Format("{0} - Disk {1} is currently in a {2} state. Type: {3} - SN: {4}", result, hdds[i].Name, hdds[i].Status, hdds[i].DriveType, hdds[i].SerialNumber)); } hce.Result = getCorrectStatusResult(hce.Result, result); } catch (Exception ex) { hce.Error.Add(string.Format("{0} - Failed to get hard drive status. Exception: {1}", StatusResult.Error, ex.Message)); hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error); } } } catch (Exception ex) { hce.Error.Add(string.Format("{0} - Failed to perform hard drive check. {1}", StatusResult.Error, ex.Message)); hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error); } #endregion CheckHardDrives hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); }
public async Task <HealthRollup> CheckWebServer(FiOSWebServer Server) { HealthRollup hru = new HealthRollup(); HealthCheckError hce = new HealthCheckError(); hru.Server = Server; hce.Result = StatusResult.Ok; hce.HCType = HealthCheckType.IIS; if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.IIS)) { hce.Result = StatusResult.Skipped; hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } if (!Server.IsOnline) { hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - Cannot communicate with web services due to the server being unreachable.", StatusResult.Critical)); hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } try { using (var iisManager = new IISServerMgr(Server)) { foreach (var site in iisManager.GetSiteCollection()) { switch (site.State) { case ObjectState.Started: break; case ObjectState.Stopped: hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - IIS Site {1} is in a stopped state.", StatusResult.Critical, site.Name)); break; case ObjectState.Stopping: hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - IIS Site {1} is currently in a stopping state.", StatusResult.Error, site.Name)); break; case ObjectState.Starting: hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - IIS Site {1} is currently starting.", StatusResult.Warning, site.Name)); break; } } } } catch (Exception ex) { hce.Result = StatusResult.Error; hce.Error.Add(string.Format("{0} - Error connecting to IIS. {1}", StatusResult.Error, ex.Message)); } hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); }