private Task <Tuple <StatusResult, string> > checkServicesAsync(string serverName, string serviceName, string serviceDesc)
{
var error = string.Empty;
var result = StatusResult.Ok;
using (var svcCtrl = new ServiceController(serviceName, serverName))
{
if (svcCtrl.Status != ServiceControllerStatus.Running)
{
if (svcCtrl.Status == ServiceControllerStatus.Stopped)
{
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Critical);
}
error = string.Format("{0} - {1} service is in a {2} state. {3}", result, serviceName, svcCtrl.Status, serviceDesc);
}
}
return(Task.FromResult <Tuple <StatusResult, string> >(new Tuple <StatusResult, string>(result, error)));
}
public async Task <HealthRollupCollection> CheckSliceFilesAsync(FiOSServer[] Servers)
{
this._progress = 1;
//var beIMGDatasources = DBConfig.GetDBs().Where(x => x.Type == DSType.TSQL && x.Role == FiOSRole.AdminConsole);
//if (beIMGDatasources.Count() < 1)
//{
// throw new Exception("No SQL datasources could be found in the configuration for Admin Console.");
//}
//var beAppDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Application)).FirstOrDefault();
//var beAdminDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Admin)).FirstOrDefault();
IEnumerable <FiOSWebServer> feIMGWebServers = null;
if (Servers == null)
{
feIMGWebServers = (await getWebServers(this.beIMGAppConnectionStr)).Where(x => x.HostFunction == ServerFunction.Web && x.HostRole == FiOSRole.IMG);
}
else
{
feIMGWebServers = Servers.Where(x => x.HostRole == FiOSRole.IMG && x.HostFunction == ServerFunction.Web).Select(x => x as FiOSWebServer);
}
var drvLetter = await getDriveLetter(this.beIMGAdminConnectionStr);
var epgRgnPaths = await getRegionPath(this.beIMGAppConnectionStr, feIMGWebServers.Select(x => x.HostLocationName).ToArray());
if (feIMGWebServers.Count() == 0)
{
throw new Exception("No front end web servers were found.");
}
if (string.IsNullOrEmpty(drvLetter))
{
throw new Exception("Could not find local drive letter.");
}
if (epgRgnPaths.Count() == 0)
{
throw new Exception("No EPG regions could be found.");
}
var currentDate = DateTime.Today;
//If the tool is run between midnight and 3am, then set the current date to yesterday
if (currentDate.Hour < 4)
{
currentDate = currentDate.AddDays(-1);
}
int totalServers = feIMGWebServers.Count();
int index = 0;
foreach (var svr in feIMGWebServers.ToList())
{
if (this.isCancelRequested)
{
continue;
}
var hru = new HealthRollup();
var hce = new HealthCheckError();
hce.HCType = HealthCheckType.EPG;
hce.Result = StatusResult.Ok;
hru.Server = svr as FiOSWebServer;
var epgRgns = epgRgnPaths.Where(x => x.VHOId == svr.HostLocationName);
string sliceDir = string.Empty, dbDir = string.Empty;
foreach (var region in epgRgns)
{
if (region.StbType == STBType.QIP)
{
string[] stbModelTypes = new string[] { "BigEndian", "LittleEndian" };
for (int i = 0; i < 2; i++)
{
sliceDir = Path.Combine(region.ApplicationPath, stbModelTypes[i]);
if (!(Directory.Exists(sliceDir)))
{
sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$");
if (!(Directory.Exists(sliceDir)))
{
hce.Error.Add(string.Format("{0} - Could not find QIP directory at {1} for region {2}.", StatusResult.Warning, sliceDir, region.VirtualChannel));
}
}
sliceDir = Directory.EnumerateDirectories(sliceDir).Where(x => x.Contains(region.VHOId) && x.Contains(region.VirtualChannel)).FirstOrDefault();
if (region.SlicerVersion == null)
{
hce.Error.Add(string.Format("{0} - QIP slicer version not found for region {1}.", StatusResult.Warning, region.VirtualChannel));
}
else if (string.IsNullOrEmpty(sliceDir))
{
hce.Error.Add(string.Format("{0} - Could not find QIP slice file directory for region {1}.", StatusResult.Warning, region.VirtualChannel));
continue;
}
sliceDir = Path.Combine(sliceDir, region.SlicerVersion);
sliceDir = @"\\" + svr.HostFullName + sliceDir.Replace(":", "$");
if (!Directory.Exists(sliceDir))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Unable to find QIP slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
continue;
}
var files = Directory.EnumerateFiles(sliceDir, "*.bin", SearchOption.AllDirectories);
//Check timestamps
var tsResult = checkSliceFileTimeStamp(files);
if (!string.IsNullOrEmpty(tsResult))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(tsResult);
}
//Check guide data day count
if (files.Count() < 15)
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Expecting 15 slice files in {1} but only found {2}. Verify there are 14 days worth of guide data for region {3}.",
StatusResult.Critical, sliceDir, files.Count(), region.RegionId));
}
}
}
else if (region.StbType == STBType.VMS)
{
dbDir = Path.Combine(region.DbFilePath, region.VHOId, region.VirtualChannel, region.SlicerVersion);
sliceDir = Path.Combine(region.ApplicationPath, region.VirtualChannel, region.SlicerVersion);
dbDir = @"\\" + svr.HostName + "\\" + dbDir.Replace(":", "$");
sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$");
bool skip = false;
if (!Directory.Exists(sliceDir))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Unable to find VMS slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
skip = true;
}
if (!Directory.Exists(dbDir))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
hce.Error.Add(string.Format("{0} - Unable to find VMS db file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
skip = true;
}
if (skip)
{
continue;
}
var sliceFiles = Directory.EnumerateFiles(sliceDir, "*.json", SearchOption.AllDirectories);
var dbFiles = Directory.EnumerateFiles(dbDir).Where(x => x.ToLower().EndsWith("json") || x.ToLower().EndsWith("gz"));
//Check timestamps
var tsResult = new List <string>()
{
checkSliceFileTimeStamp(sliceFiles), checkSliceFileTimeStamp(dbFiles)
};
tsResult.ForEach(x =>
{
if (!string.IsNullOrEmpty(x))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(x);
}
});
//Check guide data day count
//Extract all of the dates from the schedule files
var sliceSCHFiles = sliceFiles.Where(x => x.ToUpper().Contains("_SCH_")).Select(x => Path.GetFileNameWithoutExtension(x).Split('_')[2]).Distinct();
List <DateTime> sliceFileDates = new List <DateTime>();
foreach (var sliceSCHFile in sliceSCHFiles)
{
DateTime dtSliceFile = DateTime.Today;
if (!DateTime.TryParseExact(sliceSCHFile, "MMddyyyy", null, System.Globalization.DateTimeStyles.None, out dtSliceFile))
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Warning);
hce.Error.Add(string.Format("{0} - Unable to extract date from VMS epg schedule slice file {1} for region {2}.", StatusResult.Warning, sliceSCHFile, region.RegionId));
}
else
{
sliceFileDates.Add(dtSliceFile);
}
}
sliceFileDates = sliceFileDates.Distinct().ToList();
if (sliceFileDates.Any(x => x.Date < currentDate.Date))
{
var sfd = sliceFileDates.Min(x => x.Date);
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(string.Format("{0} - Old EPG data exists at {1} for region {2}. Earliest guide data date is {3} but should be {4}.", StatusResult.Error, sliceDir, region.RegionId, sfd.Date.ToString("MM/dd/yyyy"), currentDate.Date.ToString("MM/dd/yyyy")));
}
if (sliceFileDates.Count < 15)
{
hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
hce.Error.Add(string.Format("{0} - There are not 14 days worth of guide data in {1} for region {2}. Current number of days: {3}", StatusResult.Error, sliceDir, region.RegionId, sliceFileDates.Count));
}
}
}
hru.Errors.Add(hce);
this.hruCol.PutObject(hru);
this._progress = (int)(((decimal)++ index / (decimal)totalServers) * 100);
}
return(await Task.FromResult <HealthRollupCollection>(hruCol));
}
private async Task <Tuple <StatusResult, List <string> > > compareAppPoolsAsync(ApplicationPoolCollection appPools1, ApplicationPoolCollection appPools2, string server1, string server2)
{
var result = StatusResult.Ok;
var errors = new List <string>();
if (appPools1.Count < appPools2.Count)
{
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Error);
appPools2.Except(appPools1, new AppPoolComparer()).Select(x => x.Name).ToList().ForEach((ap) =>
{
errors.Add(string.Format("{0} - {1} application pool exists on {2} but does not exist on {3}", result, ap, server2, server1));
});
}
foreach (var ap in appPools1)
{
var diffAp = appPools2.Where(x => x.Name == ap.Name).FirstOrDefault();
if (diffAp == null)
{
continue;
}
switch (ap.State)
{
case ObjectState.Started:
case ObjectState.Starting:
break;
case ObjectState.Stopped:
errors.Add(string.Format("{0} - {1} application pool is in a stopped state.", StatusResult.Critical, ap.Name));
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Critical);
break;
case ObjectState.Stopping:
case ObjectState.Unknown:
errors.Add(string.Format("{0} - {1} application pool is in a {2} state", StatusResult.Warning, ap.Name, ap.State));
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Warning);
break;
}
if (!ap.AutoStart.Equals(diffAp.AutoStart))
{
errors.Add(string.Format("{0} - {1} application pool auto start config does not match with {2}. Currently set to {3}.", StatusResult.Warning, ap.Name, server2, ap.AutoStart));
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Warning);
}
if (!ap.Enable32BitAppOnWin64.Equals(diffAp.Enable32BitAppOnWin64))
{
errors.Add(string.Format("{0} - 32 bit applications is {1} on application pool {2}, but is {2} on {3}", StatusResult.Error,
ap.Enable32BitAppOnWin64 ? "enabled" : "disabled",
diffAp.Enable32BitAppOnWin64 ? "enabled" : "disabled",
server2
));
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Error);
}
if (!ap.QueueLength.Equals(diffAp.QueueLength))
{
errors.Add(string.Format("{0} - {1} application pool queue length does not match with {2}. Currently set to {3}.", StatusResult.Warning, ap.Name, server2, ap.QueueLength));
result = GenericChecks.getCorrectStatusResult(result, StatusResult.Warning);
}
var processModelResults = await compareProcessModelsAsync(ap.ProcessModel, diffAp.ProcessModel);
}
return(await Task.FromResult <Tuple <StatusResult, List <string> > >(new Tuple <StatusResult, List <string> >(result, errors)));
}
