/// <summary> /// Performs all server checks on the server /// </summary> /// <param name="Server">FiOS Server</param> /// <returns>A health rollup containing all general server checks</returns> public async Task <HealthRollup> PerformServerCheck(FiOSServer Server) { var hru = new HealthRollup(); var hce = new HealthCheckError(); hru.Server = Server; if (!Server.IsActive) { hce.Result = StatusResult.Skipped; hce.HCType = HealthCheckType.GeneralServer; if (Server.IsOnline) { hce.Error.Add("Server is marked as inactive but is currently online."); } else { hce.Error.Add("Server is offline"); } hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } if (!Server.IsOnline) { hce.Error.Add(string.Format("{0} - Server is offline or unreachable via it's FQDN.", StatusResult.Critical)); hce.Result = StatusResult.Critical; hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } #region CheckHardDrives try { var hdds = await HardDrive.GetHardDriveAsync(Server.HostFullName); for (int i = 0; i < hdds.Length; i++) { var result = StatusResult.Ok; //If server is exempt, skip if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.HardDrive, hdds[i].DriveLetter)) { hce.Error.Add(string.Format("Skipped hard drive checks for drive letter {0}. {1}GB Remaining.", hdds[i].DriveLetter, ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1"))); continue; } //Check Hard Drive Space try { result = await checkHDDSpaceAsync(hdds[i]); if (result != StatusResult.Ok) { hce.Error.Add(string.Format("{0} - Disk {1} currently only has {2}GB of {3}GB of space remaining. SN: {4}.", result, hdds[i].Name, ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1"), ((decimal)hdds[i].Capacity / 1024M / 1024M / 1024M).ToString("N1"), hdds[i].SerialNumber)); hce.Result = getCorrectStatusResult(hce.Result, result); } } catch (Exception ex) { hce.Error.Add(string.Format("{0} - Failed to get available disk space for {2}. Exception: {1}", StatusResult.Error, ex.Message, hdds[i].DriveLetter)); hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error); } //Check Hard Drive Status try { result = await checkHDDStatusAsync(hdds[i]); hce.Result = getCorrectStatusResult(hce.Result, result); if (result != StatusResult.Ok) { hce.Error.Add(string.Format("{0} - Disk {1} is currently in a {2} state. Type: {3} - SN: {4}", result, hdds[i].Name, hdds[i].Status, hdds[i].DriveType, hdds[i].SerialNumber)); } hce.Result = getCorrectStatusResult(hce.Result, result); } catch (Exception ex) { hce.Error.Add(string.Format("{0} - Failed to get hard drive status. Exception: {1}", StatusResult.Error, ex.Message)); hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error); } } } catch (Exception ex) { hce.Error.Add(string.Format("{0} - Failed to perform hard drive check. {1}", StatusResult.Error, ex.Message)); hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error); } #endregion CheckHardDrives hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); }
public async Task <HealthRollupCollection> CheckSliceFilesAsync(FiOSServer[] Servers) { this._progress = 1; //var beIMGDatasources = DBConfig.GetDBs().Where(x => x.Type == DSType.TSQL && x.Role == FiOSRole.AdminConsole); //if (beIMGDatasources.Count() < 1) //{ // throw new Exception("No SQL datasources could be found in the configuration for Admin Console."); //} //var beAppDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Application)).FirstOrDefault(); //var beAdminDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Admin)).FirstOrDefault(); IEnumerable <FiOSWebServer> feIMGWebServers = null; if (Servers == null) { feIMGWebServers = (await getWebServers(this.beIMGAppConnectionStr)).Where(x => x.HostFunction == ServerFunction.Web && x.HostRole == FiOSRole.IMG); } else { feIMGWebServers = Servers.Where(x => x.HostRole == FiOSRole.IMG && x.HostFunction == ServerFunction.Web).Select(x => x as FiOSWebServer); } var drvLetter = await getDriveLetter(this.beIMGAdminConnectionStr); var epgRgnPaths = await getRegionPath(this.beIMGAppConnectionStr, feIMGWebServers.Select(x => x.HostLocationName).ToArray()); if (feIMGWebServers.Count() == 0) { throw new Exception("No front end web servers were found."); } if (string.IsNullOrEmpty(drvLetter)) { throw new Exception("Could not find local drive letter."); } if (epgRgnPaths.Count() == 0) { throw new Exception("No EPG regions could be found."); } var currentDate = DateTime.Today; //If the tool is run between midnight and 3am, then set the current date to yesterday if (currentDate.Hour < 4) { currentDate = currentDate.AddDays(-1); } int totalServers = feIMGWebServers.Count(); int index = 0; foreach (var svr in feIMGWebServers.ToList()) { if (this.isCancelRequested) { continue; } var hru = new HealthRollup(); var hce = new HealthCheckError(); hce.HCType = HealthCheckType.EPG; hce.Result = StatusResult.Ok; hru.Server = svr as FiOSWebServer; var epgRgns = epgRgnPaths.Where(x => x.VHOId == svr.HostLocationName); string sliceDir = string.Empty, dbDir = string.Empty; foreach (var region in epgRgns) { if (region.StbType == STBType.QIP) { string[] stbModelTypes = new string[] { "BigEndian", "LittleEndian" }; for (int i = 0; i < 2; i++) { sliceDir = Path.Combine(region.ApplicationPath, stbModelTypes[i]); if (!(Directory.Exists(sliceDir))) { sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$"); if (!(Directory.Exists(sliceDir))) { hce.Error.Add(string.Format("{0} - Could not find QIP directory at {1} for region {2}.", StatusResult.Warning, sliceDir, region.VirtualChannel)); } } sliceDir = Directory.EnumerateDirectories(sliceDir).Where(x => x.Contains(region.VHOId) && x.Contains(region.VirtualChannel)).FirstOrDefault(); if (region.SlicerVersion == null) { hce.Error.Add(string.Format("{0} - QIP slicer version not found for region {1}.", StatusResult.Warning, region.VirtualChannel)); } else if (string.IsNullOrEmpty(sliceDir)) { hce.Error.Add(string.Format("{0} - Could not find QIP slice file directory for region {1}.", StatusResult.Warning, region.VirtualChannel)); continue; } sliceDir = Path.Combine(sliceDir, region.SlicerVersion); sliceDir = @"\\" + svr.HostFullName + sliceDir.Replace(":", "$"); if (!Directory.Exists(sliceDir)) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical); hce.Error.Add(string.Format("{0} - Unable to find QIP slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId)); continue; } var files = Directory.EnumerateFiles(sliceDir, "*.bin", SearchOption.AllDirectories); //Check timestamps var tsResult = checkSliceFileTimeStamp(files); if (!string.IsNullOrEmpty(tsResult)) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error); hce.Error.Add(tsResult); } //Check guide data day count if (files.Count() < 15) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical); hce.Error.Add(string.Format("{0} - Expecting 15 slice files in {1} but only found {2}. Verify there are 14 days worth of guide data for region {3}.", StatusResult.Critical, sliceDir, files.Count(), region.RegionId)); } } } else if (region.StbType == STBType.VMS) { dbDir = Path.Combine(region.DbFilePath, region.VHOId, region.VirtualChannel, region.SlicerVersion); sliceDir = Path.Combine(region.ApplicationPath, region.VirtualChannel, region.SlicerVersion); dbDir = @"\\" + svr.HostName + "\\" + dbDir.Replace(":", "$"); sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$"); bool skip = false; if (!Directory.Exists(sliceDir)) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical); hce.Error.Add(string.Format("{0} - Unable to find VMS slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId)); skip = true; } if (!Directory.Exists(dbDir)) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical); hce.Error.Add(string.Format("{0} - Unable to find VMS db file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId)); skip = true; } if (skip) { continue; } var sliceFiles = Directory.EnumerateFiles(sliceDir, "*.json", SearchOption.AllDirectories); var dbFiles = Directory.EnumerateFiles(dbDir).Where(x => x.ToLower().EndsWith("json") || x.ToLower().EndsWith("gz")); //Check timestamps var tsResult = new List <string>() { checkSliceFileTimeStamp(sliceFiles), checkSliceFileTimeStamp(dbFiles) }; tsResult.ForEach(x => { if (!string.IsNullOrEmpty(x)) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error); hce.Error.Add(x); } }); //Check guide data day count //Extract all of the dates from the schedule files var sliceSCHFiles = sliceFiles.Where(x => x.ToUpper().Contains("_SCH_")).Select(x => Path.GetFileNameWithoutExtension(x).Split('_')[2]).Distinct(); List <DateTime> sliceFileDates = new List <DateTime>(); foreach (var sliceSCHFile in sliceSCHFiles) { DateTime dtSliceFile = DateTime.Today; if (!DateTime.TryParseExact(sliceSCHFile, "MMddyyyy", null, System.Globalization.DateTimeStyles.None, out dtSliceFile)) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Warning); hce.Error.Add(string.Format("{0} - Unable to extract date from VMS epg schedule slice file {1} for region {2}.", StatusResult.Warning, sliceSCHFile, region.RegionId)); } else { sliceFileDates.Add(dtSliceFile); } } sliceFileDates = sliceFileDates.Distinct().ToList(); if (sliceFileDates.Any(x => x.Date < currentDate.Date)) { var sfd = sliceFileDates.Min(x => x.Date); hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error); hce.Error.Add(string.Format("{0} - Old EPG data exists at {1} for region {2}. Earliest guide data date is {3} but should be {4}.", StatusResult.Error, sliceDir, region.RegionId, sfd.Date.ToString("MM/dd/yyyy"), currentDate.Date.ToString("MM/dd/yyyy"))); } if (sliceFileDates.Count < 15) { hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error); hce.Error.Add(string.Format("{0} - There are not 14 days worth of guide data in {1} for region {2}. Current number of days: {3}", StatusResult.Error, sliceDir, region.RegionId, sliceFileDates.Count)); } } } hru.Errors.Add(hce); this.hruCol.PutObject(hru); this._progress = (int)(((decimal)++ index / (decimal)totalServers) * 100); } return(await Task.FromResult <HealthRollupCollection>(hruCol)); }
public static async Task <IEnumerable <HealthRollup> > CheckWindowsServices(FiOSServer Server, HCWinService[] ServicesToCheck) { var hruList = new List <HealthRollup>(); var hcErrors = new List <HealthCheckError>(); if (!Server.IsOnline || !Server.IsActive) { var svcFuncts = ServicesToCheck.GroupBy(x => x.Function) .Select(x => new { x.Key, RoleFunctions = x.Where(y => y.Function.Equals(x.Key)).SelectMany(y => y.Roles.Select(z => z.Item3)).ToList(), ServerCount = x.Where(y => y.Function.Equals(x.Key)).SelectMany(y => y.Servers).Count(), }).ToList(); svcFuncts.ForEach(x => { var hcErr = new HealthCheckError(); if (Server.IsActive) { hcErr.Result = StatusResult.Critical; } else { hcErr.Result = StatusResult.Skipped; } if (x.Key == ServerFunction.Database || (x.RoleFunctions.Count > 0 && x.RoleFunctions.Any(y => y.Equals(ServerFunction.Database)))) { hcErr.HCType = HealthCheckType.Database; } else if (x.Key == ServerFunction.Web || (x.RoleFunctions.Count > 0 && x.RoleFunctions.Any(y => y.Equals(ServerFunction.Web)))) { hcErr.HCType = HealthCheckType.IIS; } if (!(hcErr.HCType == HealthCheckType.GeneralServer) && Server.IsActive) { hcErr.Error.Add(string.Format("Cannot check windows services because the server is offline.")); } hcErrors.Add(hcErr); }); var hru = new HealthRollup() { Server = Server, Errors = hcErrors }; hruList.Add(hru); return(await Task.FromResult <IEnumerable <HealthRollup> >(hruList)); } hcErrors = await checkWindowsServices(Server.HostFullName, ServicesToCheck.Where(x => !x.OnePerGroup).ToArray()); foreach (var result in hcErrors.GroupBy(x => x.HCType).Select(x => new HealthCheckError() { HCType = x.Key, Result = x.Where(y => y.HCType.Equals(x.Key)).Select(y => y.Result).Max(), Error = x.Where(y => y.HCType.Equals(x.Key)).SelectMany(y => y.Error).ToList() })) { var hru = new HealthRollup(); hru.Server = Server; hru.Errors.Add(result); hruList.Add(hru); } return(await Task.FromResult <IEnumerable <HealthRollup> >(hruList)); }
public async Task <HealthRollup> CheckWebServer(FiOSWebServer Server) { HealthRollup hru = new HealthRollup(); HealthCheckError hce = new HealthCheckError(); hru.Server = Server; hce.Result = StatusResult.Ok; hce.HCType = HealthCheckType.IIS; if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.IIS)) { hce.Result = StatusResult.Skipped; hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } if (!Server.IsOnline) { hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - Cannot communicate with web services due to the server being unreachable.", StatusResult.Critical)); hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); } try { using (var iisManager = new IISServerMgr(Server)) { foreach (var site in iisManager.GetSiteCollection()) { switch (site.State) { case ObjectState.Started: break; case ObjectState.Stopped: hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - IIS Site {1} is in a stopped state.", StatusResult.Critical, site.Name)); break; case ObjectState.Stopping: hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - IIS Site {1} is currently in a stopping state.", StatusResult.Error, site.Name)); break; case ObjectState.Starting: hce.Result = StatusResult.Critical; hce.Error.Add(string.Format("{0} - IIS Site {1} is currently starting.", StatusResult.Warning, site.Name)); break; } } } } catch (Exception ex) { hce.Result = StatusResult.Error; hce.Error.Add(string.Format("{0} - Error connecting to IIS. {1}", StatusResult.Error, ex.Message)); } hru.Errors.Add(hce); return(await Task.FromResult <HealthRollup>(hru)); }