/// <summary>
/// Performs all server checks on the server
/// </summary>
/// <param name="Server">FiOS Server</param>
/// <returns>A health rollup containing all general server checks</returns>
public async Task <HealthRollup> PerformServerCheck(FiOSServer Server)
{
var hru = new HealthRollup();
var hce = new HealthCheckError();
hru.Server = Server;
if (!Server.IsActive)
{
hce.Result = StatusResult.Skipped;
hce.HCType = HealthCheckType.GeneralServer;
if (Server.IsOnline)
{
hce.Error.Add("Server is marked as inactive but is currently online.");
}
else
{
hce.Error.Add("Server is offline");
}
hru.Errors.Add(hce);
return(await Task.FromResult <HealthRollup>(hru));
}
if (!Server.IsOnline)
{
hce.Error.Add(string.Format("{0} - Server is offline or unreachable via it's FQDN.", StatusResult.Critical));
hce.Result = StatusResult.Critical;
hru.Errors.Add(hce);
return(await Task.FromResult <HealthRollup>(hru));
}
#region CheckHardDrives
try
{
var hdds = await HardDrive.GetHardDriveAsync(Server.HostFullName);
for (int i = 0; i < hdds.Length; i++)
{
var result = StatusResult.Ok;
//If server is exempt, skip
if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.HardDrive, hdds[i].DriveLetter))
{
hce.Error.Add(string.Format("Skipped hard drive checks for drive letter {0}. {1}GB Remaining.", hdds[i].DriveLetter, ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1")));
continue;
}
//Check Hard Drive Space
try
{
result = await checkHDDSpaceAsync(hdds[i]);
if (result != StatusResult.Ok)
{
hce.Error.Add(string.Format("{0} - Disk {1} currently only has {2}GB of {3}GB of space remaining. SN: {4}.",
result,
hdds[i].Name,
((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1"),
((decimal)hdds[i].Capacity / 1024M / 1024M / 1024M).ToString("N1"),
hdds[i].SerialNumber));
hce.Result = getCorrectStatusResult(hce.Result, result);
}
}
catch (Exception ex)
{
hce.Error.Add(string.Format("{0} - Failed to get available disk space for {2}. Exception: {1}", StatusResult.Error, ex.Message, hdds[i].DriveLetter));
hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error);
}
//Check Hard Drive Status
try
{
result = await checkHDDStatusAsync(hdds[i]);
hce.Result = getCorrectStatusResult(hce.Result, result);
if (result != StatusResult.Ok)
{
hce.Error.Add(string.Format("{0} - Disk {1} is currently in a {2} state. Type: {3} - SN: {4}", result, hdds[i].Name, hdds[i].Status, hdds[i].DriveType, hdds[i].SerialNumber));
}
hce.Result = getCorrectStatusResult(hce.Result, result);
}
catch (Exception ex)
{
hce.Error.Add(string.Format("{0} - Failed to get hard drive status. Exception: {1}", StatusResult.Error, ex.Message));
hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error);
}
}
}
catch (Exception ex)
{
hce.Error.Add(string.Format("{0} - Failed to perform hard drive check. {1}", StatusResult.Error, ex.Message));
hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error);
}
#endregion CheckHardDrives
hru.Errors.Add(hce);
return(await Task.FromResult <HealthRollup>(hru));
}
/// <summary>
/// Checks windows services.
/// </summary>
/// <param name="ServerName">Name of the server being checked</param>
/// <param name="ServicesToCheck">Array of services to check</param>
/// <returns>A tuple containing the highest level of status result, and a list of errors.</returns>
private static Task <List <HealthCheckError> > checkWindowsServices(string ServerName, HCWinService[] ServicesToCheck)
{
var errors = new List <HealthCheckError>();
var exceptions = new List <Exception>();
for (int i = 0; i < ServicesToCheck.Length; i++)
{
try
{
var hcErr = new HealthCheckError();
hcErr.Result = StatusResult.Ok;
using (var svc = new ServiceController(ServicesToCheck[i].Name, ServerName))
{
StringBuilder sb = new StringBuilder();
if (ServicesToCheck[i].Function == ServerFunction.Database || (ServicesToCheck[i].Roles.Count > 0 && ServicesToCheck[i].Roles.Any(x => x.Item3.Equals(ServerFunction.Database))))
{
hcErr.HCType = HealthCheckType.Database;
}
else if (ServicesToCheck[i].Function == ServerFunction.Web || (ServicesToCheck[i].Roles.Count > 0 && ServicesToCheck[i].Roles.Any(x => x.Item3.Equals(ServerFunction.Web))))
{
hcErr.HCType = HealthCheckType.IIS;
}
//Check Status
try
{
if (ServicesToCheck[i].CheckStatus.Count > 0 && !ServicesToCheck[i].CheckStatus.Contains(svc.Status))
{
hcErr.Result = getCorrectStatusResult(hcErr.Result, StatusResult.Error);
sb.AppendFormat("{0} - Windows service \"{1}\" is in a {2} state.", StatusResult.Error, svc.ServiceName, svc.Status);
sb.AppendFormat(" Expected state(s): {0}.", string.Join(",", ServicesToCheck[i].CheckStatus));
hcErr.Error.Add(sb.ToString());
}
}
catch (Exception ex)
{
hcErr.Result = getCorrectStatusResult(hcErr.Result, StatusResult.Critical);
hcErr.Error.Add(string.Format("{0} - Failed to check windows service status for \"{1}\" on \"{2}\". {3}", StatusResult.Critical, ServicesToCheck[i].Name, ServerName, ex.Message));
if (ex.Message.ToUpper().Contains("WAS NOT FOUND"))
{
errors.Add(hcErr);
continue;
}
}
//Check StartupType. Can be deprecated after .net upgrade, and instead use the ServiceController.
try
{
if (ServicesToCheck[i].CheckStartupType.Count > 0)
{
var stType = GetWindowsServiceStartMode(ServerName, svc.ServiceName);
if (!ServicesToCheck[i].CheckStartupType.Contains(stType))
{
hcErr.Result = getCorrectStatusResult(hcErr.Result, StatusResult.Warning);
sb.Clear();
sb.AppendFormat("{0} - Windows service \"{1}\" startup type is set to {2}.", StatusResult.Warning, svc.ServiceName, stType);
sb.AppendFormat(" Expected startup type(s): {0}", string.Join(",", ServicesToCheck[i].CheckStartupType));
hcErr.Error.Add(sb.ToString());
}
}
}
catch (Exception ex)
{
hcErr.Result = getCorrectStatusResult(hcErr.Result, StatusResult.Critical);
hcErr.Error.Add(string.Format("{0} - Failed to check windows service \"{1}\" on \"{2}\". {3}", StatusResult.Critical, ServicesToCheck[i].Name, ServerName, ex.Message));
errors.Add(hcErr);
continue;
}
//Check LogonAs Account.
try
{
if (ServicesToCheck[i].CheckLogonAs.Count > 0)
{
var logonAcct = Toolset.RemoveWhitespace(GetWindowsServiceLogonAccount(ServerName, svc.ServiceName)).ToUpper();
if (string.IsNullOrEmpty(logonAcct))
{
logonAcct = "Unknown";
}
if (!ServicesToCheck[i].CheckLogonAs.Contains(logonAcct))
{
hcErr.Result = getCorrectStatusResult(hcErr.Result, StatusResult.Warning);
sb.Clear();
sb.AppendFormat("{0} - Windows service \"{1}\" logon account is {2}.", StatusResult.Warning, svc.ServiceName, logonAcct);
sb.AppendFormat(" Expected logon account(s): {0}", string.Join(",", ServicesToCheck[i].CheckLogonAs));
hcErr.Error.Add(sb.ToString());
}
}
}
catch (Exception ex)
{
hcErr.Result = getCorrectStatusResult(hcErr.Result, StatusResult.Critical);
hcErr.Error.Add(string.Format("{0} - Failed to check windows service logon account for \"{1}\" on \"{2}\". {3}", StatusResult.Critical, ServicesToCheck[i].Name, ServerName, ex.Message));
}
}
errors.Add(hcErr);
}
catch (Exception ex)
{
exceptions.Add(new Exception(string.Format("{0} - Failed to get windows service \"{1}\" on {2}. {3}", StatusResult.Critical, ServicesToCheck[i].Name, ServerName, ex.Message), ex.InnerException));
}
}
if (exceptions.Count > 0)
{
var hce = new HealthCheckError();
exceptions.ForEach(x => hce.Error.Add(x.Message));
hce.HCType = HealthCheckType.GeneralServer;
hce.Result = StatusResult.Critical;
errors.Add(hce);
}
return(Task.FromResult <List <HealthCheckError> >(errors));
}
public async Task <HealthRollupCollection> CheckSliceFilesAsync(FiOSServer[] Servers)
{
this._progress = 1;
//var beIMGDatasources = DBConfig.GetDBs().Where(x => x.Type == DSType.TSQL && x.Role == FiOSRole.AdminConsole);
//if (beIMGDatasources.Count() < 1)
//{
// throw new Exception("No SQL datasources could be found in the configuration for Admin Console.");
//}
//var beAppDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Application)).FirstOrDefault();
//var beAdminDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Admin)).FirstOrDefault();
IEnumerable <FiOSWebServer> feIMGWebServers = null;
if (Servers == null)
{
feIMGWebServers = (await getWebServers(this.beIMGAppConnectionStr)).Where(x => x.HostFunction == ServerFunction.Web && x.HostRole == FiOSRole.IMG);
}
else
{
feIMGWebServers = Servers.Where(x => x.HostRole == FiOSRole.IMG && x.HostFunction == ServerFunction.Web).Select(x => x as FiOSWebServer);
}
var drvLetter = await getDriveLetter(this.beIMGAdminConnectionStr);
var epgRgnPaths = await getRegionPath(this.beIMGAppConnectionStr, feIMGWebServers.Select(x => x.HostLocationName).ToArray());
if (feIMGWebServers.Count() == 0)
{
throw new Exception("No front end web servers were found.");
}
if (string.IsNullOrEmpty(drvLetter))
{
throw new Exception("Could not find local drive letter.");
}
if (epgRgnPaths.Count() == 0)
{
throw new Exception("No EPG regions could be found.");
}
var currentDate = DateTime.Today;
//If the tool is run between midnight and 3am, then set the current date to yesterday
if (currentDate.Hour < 4)
{
currentDate = currentDate.AddDays(-1);
}
int totalServers = feIMGWebServers.Count();
int index = 0;
foreach (var svr in feIMGWebServers.ToList())
{
if (this.isCancelRequested)
{
continue;
}
var hru = new HealthRollup();
var hce = new HealthCheckError();
hce.HCType = HealthCheckType.EPG;
hce.Result = StatusResult.Ok;
hru.Server = svr as FiOSWebServer;
var epgRgns = epgRgnPaths.Where(x => x.VHOId == svr.HostLocationName);
string sliceDir = string.Empty, dbDir = string.Empty;
foreach (var region in epgRgns)
{
if (region.StbType == STBType.QIP)
{
string[] stbModelTypes = new string[] { "BigEndian", "LittleEndian" };
for (int i = 0; i < 2; i++)
{
sliceDir = Path.Combine(region.ApplicationPath, stbModelTypes[i]);
if (!(Directory.Exists(sliceDir)))
{
sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$");
if (!(Directory.Exists(sliceDir)))
{
hce.Error.Add(string.Format("{0} - Could not find QIP directory at {1} for region {2}.", StatusResult.Warning, sliceDir, region.VirtualChannel));
}
}
sliceDir = Directory.EnumerateDirectories(sliceDir).Where(x => x.Contains(region.VHOId) && x.Contains(region.VirtualChannel)).FirstOrDefault();
if (region.SlicerVersion == null)
{
hce.Error.Add(string.Format("{0} - QIP slicer version not found for region {1}.", StatusResult.Warning, region.VirtualChannel));
}
else if (string.IsNullOrEmpty(sliceDir))
{
hce.Error.Add(string.Format("{0} - Could not find QIP slice file directory for region {1}.", StatusResult.Warning, region.VirtualChannel));
continue;
}
sliceDir = Path.Combine(sliceDir, region.SlicerVersion);
sliceDir = @"\\" + svr.HostFullName + sliceDir.Replace(":", "$");
if (!Directory.Exists(sliceDir))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Unable to find QIP slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
continue;
}
var files = Directory.EnumerateFiles(sliceDir, "*.bin", SearchOption.AllDirectories);
//Check timestamps
var tsResult = checkSliceFileTimeStamp(files);
if (!string.IsNullOrEmpty(tsResult))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(tsResult);
}
//Check guide data day count
if (files.Count() < 15)
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Expecting 15 slice files in {1} but only found {2}. Verify there are 14 days worth of guide data for region {3}.",
StatusResult.Critical, sliceDir, files.Count(), region.RegionId));
}
}
}
else if (region.StbType == STBType.VMS)
{
dbDir = Path.Combine(region.DbFilePath, region.VHOId, region.VirtualChannel, region.SlicerVersion);
sliceDir = Path.Combine(region.ApplicationPath, region.VirtualChannel, region.SlicerVersion);
dbDir = @"\\" + svr.HostName + "\\" + dbDir.Replace(":", "$");
sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$");
bool skip = false;
if (!Directory.Exists(sliceDir))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Unable to find VMS slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
skip = true;
}
if (!Directory.Exists(dbDir))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Unable to find VMS db file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
skip = true;
}
if (skip)
{
continue;
}
var sliceFiles = Directory.EnumerateFiles(sliceDir, "*.json", SearchOption.AllDirectories);
var dbFiles = Directory.EnumerateFiles(dbDir).Where(x => x.ToLower().EndsWith("json") || x.ToLower().EndsWith("gz"));
//Check timestamps
var tsResult = new List <string>()
{
checkSliceFileTimeStamp(sliceFiles), checkSliceFileTimeStamp(dbFiles)
};
tsResult.ForEach(x =>
{
if (!string.IsNullOrEmpty(x))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(x);
}
});
//Check guide data day count
//Extract all of the dates from the schedule files
var sliceSCHFiles = sliceFiles.Where(x => x.ToUpper().Contains("_SCH_")).Select(x => Path.GetFileNameWithoutExtension(x).Split('_')[2]).Distinct();
List <DateTime> sliceFileDates = new List <DateTime>();
foreach (var sliceSCHFile in sliceSCHFiles)
{
DateTime dtSliceFile = DateTime.Today;
if (!DateTime.TryParseExact(sliceSCHFile, "MMddyyyy", null, System.Globalization.DateTimeStyles.None, out dtSliceFile))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Warning);
hce.Error.Add(string.Format("{0} - Unable to extract date from VMS epg schedule slice file {1} for region {2}.", StatusResult.Warning, sliceSCHFile, region.RegionId));
}
else
{
sliceFileDates.Add(dtSliceFile);
}
}
sliceFileDates = sliceFileDates.Distinct().ToList();
if (sliceFileDates.Any(x => x.Date < currentDate.Date))
{
var sfd = sliceFileDates.Min(x => x.Date);
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(string.Format("{0} - Old EPG data exists at {1} for region {2}. Earliest guide data date is {3} but should be {4}.", StatusResult.Error, sliceDir, region.RegionId, sfd.Date.ToString("MM/dd/yyyy"), currentDate.Date.ToString("MM/dd/yyyy")));
}
if (sliceFileDates.Count < 15)
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(string.Format("{0} - There are not 14 days worth of guide data in {1} for region {2}. Current number of days: {3}", StatusResult.Error, sliceDir, region.RegionId, sliceFileDates.Count));
}
}
}
hru.Errors.Add(hce);
this.hruCol.PutObject(hru);
this._progress = (int)(((decimal)++ index / (decimal)totalServers) * 100);
}
return(await Task.FromResult <HealthRollupCollection>(hruCol));
}
public static async Task <IEnumerable <HealthRollup> > CheckWindowsServices(FiOSServer Server, HCWinService[] ServicesToCheck)
{
var hruList = new List <HealthRollup>();
var hcErrors = new List <HealthCheckError>();
if (!Server.IsOnline || !Server.IsActive)
{
var svcFuncts = ServicesToCheck.GroupBy(x => x.Function)
.Select(x => new
{
x.Key,
RoleFunctions = x.Where(y => y.Function.Equals(x.Key)).SelectMany(y => y.Roles.Select(z => z.Item3)).ToList(),
ServerCount = x.Where(y => y.Function.Equals(x.Key)).SelectMany(y => y.Servers).Count(),
}).ToList();
svcFuncts.ForEach(x =>
{
var hcErr = new HealthCheckError();
if (Server.IsActive)
{
hcErr.Result = StatusResult.Critical;
}
else
{
hcErr.Result = StatusResult.Skipped;
}
if (x.Key == ServerFunction.Database || (x.RoleFunctions.Count > 0 && x.RoleFunctions.Any(y => y.Equals(ServerFunction.Database))))
{
hcErr.HCType = HealthCheckType.Database;
}
else if (x.Key == ServerFunction.Web || (x.RoleFunctions.Count > 0 && x.RoleFunctions.Any(y => y.Equals(ServerFunction.Web))))
{
hcErr.HCType = HealthCheckType.IIS;
}
if (!(hcErr.HCType == HealthCheckType.GeneralServer) && Server.IsActive)
{
hcErr.Error.Add(string.Format("Cannot check windows services because the server is offline."));
}
hcErrors.Add(hcErr);
});
var hru = new HealthRollup()
{
Server = Server, Errors = hcErrors
};
hruList.Add(hru);
return(await Task.FromResult <IEnumerable <HealthRollup> >(hruList));
}
hcErrors = await checkWindowsServices(Server.HostFullName, ServicesToCheck.Where(x => !x.OnePerGroup).ToArray());
foreach (var result in hcErrors.GroupBy(x => x.HCType).Select(x => new HealthCheckError()
{
HCType = x.Key,
Result = x.Where(y => y.HCType.Equals(x.Key)).Select(y => y.Result).Max(),
Error = x.Where(y => y.HCType.Equals(x.Key)).SelectMany(y => y.Error).ToList()
}))
{
var hru = new HealthRollup();
hru.Server = Server;
hru.Errors.Add(result);
hruList.Add(hru);
}
return(await Task.FromResult <IEnumerable <HealthRollup> >(hruList));
}
public async Task <HealthRollup> CheckWebServer(FiOSWebServer Server)
{
HealthRollup hru = new HealthRollup();
HealthCheckError hce = new HealthCheckError();
hru.Server = Server;
hce.Result = StatusResult.Ok;
hce.HCType = HealthCheckType.IIS;
if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.IIS))
{
hce.Result = StatusResult.Skipped;
hru.Errors.Add(hce);
return(await Task.FromResult <HealthRollup>(hru));
}
if (!Server.IsOnline)
{
hce.Result = StatusResult.Critical;
hce.Error.Add(string.Format("{0} - Cannot communicate with web services due to the server being unreachable.", StatusResult.Critical));
hru.Errors.Add(hce);
return(await Task.FromResult <HealthRollup>(hru));
}
try
{
using (var iisManager = new IISServerMgr(Server))
{
foreach (var site in iisManager.GetSiteCollection())
{
switch (site.State)
{
case ObjectState.Started:
break;
case ObjectState.Stopped:
hce.Result = StatusResult.Critical;
hce.Error.Add(string.Format("{0} - IIS Site {1} is in a stopped state.", StatusResult.Critical, site.Name));
break;
case ObjectState.Stopping:
hce.Result = StatusResult.Critical;
hce.Error.Add(string.Format("{0} - IIS Site {1} is currently in a stopping state.", StatusResult.Error, site.Name));
break;
case ObjectState.Starting:
hce.Result = StatusResult.Critical;
hce.Error.Add(string.Format("{0} - IIS Site {1} is currently starting.", StatusResult.Warning, site.Name));
break;
}
}
}
}
catch (Exception ex)
{
hce.Result = StatusResult.Error;
hce.Error.Add(string.Format("{0} - Error connecting to IIS. {1}", StatusResult.Error, ex.Message));
}
hru.Errors.Add(hce);
return(await Task.FromResult <HealthRollup>(hru));
}
