Exemplo n.º 1
0
        /// <summary>
        /// Performs all server checks on the server
        /// </summary>
        /// <param name="Server">FiOS Server</param>
        /// <returns>A health rollup containing all general server checks</returns>
        public async Task <HealthRollup> PerformServerCheck(FiOSServer Server)
        {
            var hru = new HealthRollup();
            var hce = new HealthCheckError();

            hru.Server = Server;

            if (!Server.IsActive)
            {
                hce.Result = StatusResult.Skipped;
                hce.HCType = HealthCheckType.GeneralServer;
                if (Server.IsOnline)
                {
                    hce.Error.Add("Server is marked as inactive but is currently online.");
                }
                else
                {
                    hce.Error.Add("Server is offline");
                }

                hru.Errors.Add(hce);
                return(await Task.FromResult <HealthRollup>(hru));
            }

            if (!Server.IsOnline)
            {
                hce.Error.Add(string.Format("{0} - Server is offline or unreachable via it's FQDN.", StatusResult.Critical));
                hce.Result = StatusResult.Critical;
                hru.Errors.Add(hce);
                return(await Task.FromResult <HealthRollup>(hru));
            }

            #region CheckHardDrives
            try
            {
                var hdds = await HardDrive.GetHardDriveAsync(Server.HostFullName);

                for (int i = 0; i < hdds.Length; i++)
                {
                    var result = StatusResult.Ok;

                    //If server is exempt, skip
                    if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.HardDrive, hdds[i].DriveLetter))
                    {
                        hce.Error.Add(string.Format("Skipped hard drive checks for drive letter {0}. {1}GB Remaining.", hdds[i].DriveLetter, ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1")));
                        continue;
                    }

                    //Check Hard Drive Space
                    try
                    {
                        result = await checkHDDSpaceAsync(hdds[i]);

                        if (result != StatusResult.Ok)
                        {
                            hce.Error.Add(string.Format("{0} - Disk {1} currently only has {2}GB of {3}GB of space remaining. SN: {4}.",
                                                        result,
                                                        hdds[i].Name,
                                                        ((decimal)hdds[i].FreeSpace / 1024M / 1024M / 1024M).ToString("N1"),
                                                        ((decimal)hdds[i].Capacity / 1024M / 1024M / 1024M).ToString("N1"),
                                                        hdds[i].SerialNumber));

                            hce.Result = getCorrectStatusResult(hce.Result, result);
                        }
                    }
                    catch (Exception ex)
                    {
                        hce.Error.Add(string.Format("{0} - Failed to get available disk space for {2}. Exception: {1}", StatusResult.Error, ex.Message, hdds[i].DriveLetter));
                        hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error);
                    }

                    //Check Hard Drive Status
                    try
                    {
                        result = await checkHDDStatusAsync(hdds[i]);

                        hce.Result = getCorrectStatusResult(hce.Result, result);

                        if (result != StatusResult.Ok)
                        {
                            hce.Error.Add(string.Format("{0} - Disk {1} is currently in a {2} state. Type: {3} - SN: {4}", result, hdds[i].Name, hdds[i].Status, hdds[i].DriveType, hdds[i].SerialNumber));
                        }

                        hce.Result = getCorrectStatusResult(hce.Result, result);
                    }
                    catch (Exception ex)
                    {
                        hce.Error.Add(string.Format("{0} - Failed to get hard drive status. Exception: {1}", StatusResult.Error, ex.Message));
                        hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error);
                    }
                }
            }
            catch (Exception ex)
            {
                hce.Error.Add(string.Format("{0} - Failed to perform hard drive check. {1}", StatusResult.Error, ex.Message));
                hce.Result = getCorrectStatusResult(hce.Result, StatusResult.Error);
            }
            #endregion CheckHardDrives

            hru.Errors.Add(hce);

            return(await Task.FromResult <HealthRollup>(hru));
        }
Exemplo n.º 2
0
        public async Task <HealthRollupCollection> CheckSliceFilesAsync(FiOSServer[] Servers)
        {
            this._progress = 1;

            //var beIMGDatasources = DBConfig.GetDBs().Where(x => x.Type == DSType.TSQL && x.Role == FiOSRole.AdminConsole);

            //if (beIMGDatasources.Count() < 1)
            //{
            //    throw new Exception("No SQL datasources could be found in the configuration for Admin Console.");
            //}

            //var beAppDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Application)).FirstOrDefault();
            //var beAdminDb = beIMGDatasources.SelectMany(x => x.Databases.Where(y => y.Location == FiOSLocation.VHE && y.Function == DbFunction.Admin)).FirstOrDefault();

            IEnumerable <FiOSWebServer> feIMGWebServers = null;

            if (Servers == null)
            {
                feIMGWebServers = (await getWebServers(this.beIMGAppConnectionStr)).Where(x => x.HostFunction == ServerFunction.Web && x.HostRole == FiOSRole.IMG);
            }
            else
            {
                feIMGWebServers = Servers.Where(x => x.HostRole == FiOSRole.IMG && x.HostFunction == ServerFunction.Web).Select(x => x as FiOSWebServer);
            }

            var drvLetter = await getDriveLetter(this.beIMGAdminConnectionStr);

            var epgRgnPaths = await getRegionPath(this.beIMGAppConnectionStr, feIMGWebServers.Select(x => x.HostLocationName).ToArray());

            if (feIMGWebServers.Count() == 0)
            {
                throw new Exception("No front end web servers were found.");
            }

            if (string.IsNullOrEmpty(drvLetter))
            {
                throw new Exception("Could not find local drive letter.");
            }

            if (epgRgnPaths.Count() == 0)
            {
                throw new Exception("No EPG regions could be found.");
            }

            var currentDate = DateTime.Today;

            //If the tool is run between midnight and 3am, then set the current date to yesterday
            if (currentDate.Hour < 4)
            {
                currentDate = currentDate.AddDays(-1);
            }

            int totalServers = feIMGWebServers.Count();
            int index        = 0;


            foreach (var svr in feIMGWebServers.ToList())
            {
                if (this.isCancelRequested)
                {
                    continue;
                }
                var hru = new HealthRollup();
                var hce = new HealthCheckError();
                hce.HCType = HealthCheckType.EPG;
                hce.Result = StatusResult.Ok;
                hru.Server = svr as FiOSWebServer;

                var epgRgns = epgRgnPaths.Where(x => x.VHOId == svr.HostLocationName);

                string sliceDir = string.Empty, dbDir = string.Empty;

                foreach (var region in epgRgns)
                {
                    if (region.StbType == STBType.QIP)
                    {
                        string[] stbModelTypes = new string[] { "BigEndian", "LittleEndian" };

                        for (int i = 0; i < 2; i++)
                        {
                            sliceDir = Path.Combine(region.ApplicationPath, stbModelTypes[i]);
                            if (!(Directory.Exists(sliceDir)))
                            {
                                sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$");
                                if (!(Directory.Exists(sliceDir)))
                                {
                                    hce.Error.Add(string.Format("{0} - Could not find QIP directory at {1} for region {2}.", StatusResult.Warning, sliceDir, region.VirtualChannel));
                                }
                            }
                            sliceDir = Directory.EnumerateDirectories(sliceDir).Where(x => x.Contains(region.VHOId) && x.Contains(region.VirtualChannel)).FirstOrDefault();

                            if (region.SlicerVersion == null)
                            {
                                hce.Error.Add(string.Format("{0} - QIP slicer version not found for region {1}.", StatusResult.Warning, region.VirtualChannel));
                            }
                            else if (string.IsNullOrEmpty(sliceDir))
                            {
                                hce.Error.Add(string.Format("{0} - Could not find QIP slice file directory for region {1}.", StatusResult.Warning, region.VirtualChannel));
                                continue;
                            }

                            sliceDir = Path.Combine(sliceDir, region.SlicerVersion);
                            sliceDir = @"\\" + svr.HostFullName + sliceDir.Replace(":", "$");

                            if (!Directory.Exists(sliceDir))
                            {
                                hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
                                hce.Error.Add(string.Format("{0} - Unable to find QIP slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
                                continue;
                            }

                            var files = Directory.EnumerateFiles(sliceDir, "*.bin", SearchOption.AllDirectories);

                            //Check timestamps
                            var tsResult = checkSliceFileTimeStamp(files);
                            if (!string.IsNullOrEmpty(tsResult))
                            {
                                hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
                                hce.Error.Add(tsResult);
                            }

                            //Check guide data day count
                            if (files.Count() < 15)
                            {
                                hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
                                hce.Error.Add(string.Format("{0} - Expecting 15 slice files in {1} but only found {2}. Verify there are 14 days worth of guide data for region {3}.",
                                                            StatusResult.Critical, sliceDir, files.Count(), region.RegionId));
                            }
                        }
                    }
                    else if (region.StbType == STBType.VMS)
                    {
                        dbDir    = Path.Combine(region.DbFilePath, region.VHOId, region.VirtualChannel, region.SlicerVersion);
                        sliceDir = Path.Combine(region.ApplicationPath, region.VirtualChannel, region.SlicerVersion);

                        dbDir    = @"\\" + svr.HostName + "\\" + dbDir.Replace(":", "$");
                        sliceDir = @"\\" + svr.HostName + "\\" + sliceDir.Replace(":", "$");

                        bool skip = false;
                        if (!Directory.Exists(sliceDir))
                        {
                            hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
                            hce.Error.Add(string.Format("{0} - Unable to find VMS slice file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
                            skip = true;
                        }

                        if (!Directory.Exists(dbDir))
                        {
                            hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Critical);
                            hce.Error.Add(string.Format("{0} - Unable to find VMS db file directory at path: {1} for region {2}.", StatusResult.Critical, sliceDir, region.RegionId));
                            skip = true;
                        }

                        if (skip)
                        {
                            continue;
                        }

                        var sliceFiles = Directory.EnumerateFiles(sliceDir, "*.json", SearchOption.AllDirectories);
                        var dbFiles    = Directory.EnumerateFiles(dbDir).Where(x => x.ToLower().EndsWith("json") || x.ToLower().EndsWith("gz"));

                        //Check timestamps
                        var tsResult = new List <string>()
                        {
                            checkSliceFileTimeStamp(sliceFiles), checkSliceFileTimeStamp(dbFiles)
                        };

                        tsResult.ForEach(x =>
                        {
                            if (!string.IsNullOrEmpty(x))
                            {
                                hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
                                hce.Error.Add(x);
                            }
                        });

                        //Check guide data day count
                        //Extract all of the dates from the schedule files
                        var             sliceSCHFiles  = sliceFiles.Where(x => x.ToUpper().Contains("_SCH_")).Select(x => Path.GetFileNameWithoutExtension(x).Split('_')[2]).Distinct();
                        List <DateTime> sliceFileDates = new List <DateTime>();

                        foreach (var sliceSCHFile in sliceSCHFiles)
                        {
                            DateTime dtSliceFile = DateTime.Today;
                            if (!DateTime.TryParseExact(sliceSCHFile, "MMddyyyy", null, System.Globalization.DateTimeStyles.None, out dtSliceFile))
                            {
                                hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Warning);
                                hce.Error.Add(string.Format("{0} - Unable to extract date from VMS epg schedule slice file {1} for region {2}.", StatusResult.Warning, sliceSCHFile, region.RegionId));
                            }
                            else
                            {
                                sliceFileDates.Add(dtSliceFile);
                            }
                        }

                        sliceFileDates = sliceFileDates.Distinct().ToList();

                        if (sliceFileDates.Any(x => x.Date < currentDate.Date))
                        {
                            var sfd = sliceFileDates.Min(x => x.Date);
                            hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
                            hce.Error.Add(string.Format("{0} - Old EPG data exists at {1} for region {2}. Earliest guide data date is {3} but should be {4}.", StatusResult.Error, sliceDir, region.RegionId, sfd.Date.ToString("MM/dd/yyyy"), currentDate.Date.ToString("MM/dd/yyyy")));
                        }

                        if (sliceFileDates.Count < 15)
                        {
                            hce.Result = GenericChecks.getCorrectStatusResult(hce.Result, StatusResult.Error);
                            hce.Error.Add(string.Format("{0} - There are not 14 days worth of guide data in {1} for region {2}. Current number of days: {3}", StatusResult.Error, sliceDir, region.RegionId, sliceFileDates.Count));
                        }
                    }
                }

                hru.Errors.Add(hce);
                this.hruCol.PutObject(hru);
                this._progress = (int)(((decimal)++ index / (decimal)totalServers) * 100);
            }

            return(await Task.FromResult <HealthRollupCollection>(hruCol));
        }
Exemplo n.º 3
0
        public static async Task <IEnumerable <HealthRollup> > CheckWindowsServices(FiOSServer Server, HCWinService[] ServicesToCheck)
        {
            var hruList = new List <HealthRollup>();

            var hcErrors = new List <HealthCheckError>();

            if (!Server.IsOnline || !Server.IsActive)
            {
                var svcFuncts = ServicesToCheck.GroupBy(x => x.Function)
                                .Select(x => new
                {
                    x.Key,
                    RoleFunctions = x.Where(y => y.Function.Equals(x.Key)).SelectMany(y => y.Roles.Select(z => z.Item3)).ToList(),
                    ServerCount   = x.Where(y => y.Function.Equals(x.Key)).SelectMany(y => y.Servers).Count(),
                }).ToList();

                svcFuncts.ForEach(x =>
                {
                    var hcErr = new HealthCheckError();

                    if (Server.IsActive)
                    {
                        hcErr.Result = StatusResult.Critical;
                    }
                    else
                    {
                        hcErr.Result = StatusResult.Skipped;
                    }

                    if (x.Key == ServerFunction.Database || (x.RoleFunctions.Count > 0 && x.RoleFunctions.Any(y => y.Equals(ServerFunction.Database))))
                    {
                        hcErr.HCType = HealthCheckType.Database;
                    }
                    else if (x.Key == ServerFunction.Web || (x.RoleFunctions.Count > 0 && x.RoleFunctions.Any(y => y.Equals(ServerFunction.Web))))
                    {
                        hcErr.HCType = HealthCheckType.IIS;
                    }

                    if (!(hcErr.HCType == HealthCheckType.GeneralServer) && Server.IsActive)
                    {
                        hcErr.Error.Add(string.Format("Cannot check windows services because the server is offline."));
                    }
                    hcErrors.Add(hcErr);
                });
                var hru = new HealthRollup()
                {
                    Server = Server, Errors = hcErrors
                };
                hruList.Add(hru);
                return(await Task.FromResult <IEnumerable <HealthRollup> >(hruList));
            }


            hcErrors = await checkWindowsServices(Server.HostFullName, ServicesToCheck.Where(x => !x.OnePerGroup).ToArray());

            foreach (var result in hcErrors.GroupBy(x => x.HCType).Select(x => new HealthCheckError()
            {
                HCType = x.Key,
                Result = x.Where(y => y.HCType.Equals(x.Key)).Select(y => y.Result).Max(),
                Error = x.Where(y => y.HCType.Equals(x.Key)).SelectMany(y => y.Error).ToList()
            }))
            {
                var hru = new HealthRollup();
                hru.Server = Server;
                hru.Errors.Add(result);
                hruList.Add(hru);
            }

            return(await Task.FromResult <IEnumerable <HealthRollup> >(hruList));
        }
Exemplo n.º 4
0
        public async Task <HealthRollup> CheckWebServer(FiOSWebServer Server)
        {
            HealthRollup     hru = new HealthRollup();
            HealthCheckError hce = new HealthCheckError();

            hru.Server = Server;
            hce.Result = StatusResult.Ok;
            hce.HCType = HealthCheckType.IIS;

            if (ServerHealthCheckConfigMgr.IsExempt(Server, ExemptionType.IIS))
            {
                hce.Result = StatusResult.Skipped;
                hru.Errors.Add(hce);
                return(await Task.FromResult <HealthRollup>(hru));
            }

            if (!Server.IsOnline)
            {
                hce.Result = StatusResult.Critical;
                hce.Error.Add(string.Format("{0} - Cannot communicate with web services due to the server being unreachable.", StatusResult.Critical));
                hru.Errors.Add(hce);
                return(await Task.FromResult <HealthRollup>(hru));
            }

            try
            {
                using (var iisManager = new IISServerMgr(Server))
                {
                    foreach (var site in iisManager.GetSiteCollection())
                    {
                        switch (site.State)
                        {
                        case ObjectState.Started:
                            break;

                        case ObjectState.Stopped:
                            hce.Result = StatusResult.Critical;
                            hce.Error.Add(string.Format("{0} - IIS Site {1} is in a stopped state.", StatusResult.Critical, site.Name));
                            break;

                        case ObjectState.Stopping:
                            hce.Result = StatusResult.Critical;
                            hce.Error.Add(string.Format("{0} - IIS Site {1} is currently in a stopping state.", StatusResult.Error, site.Name));
                            break;

                        case ObjectState.Starting:
                            hce.Result = StatusResult.Critical;
                            hce.Error.Add(string.Format("{0} - IIS Site {1} is currently starting.", StatusResult.Warning, site.Name));
                            break;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                hce.Result = StatusResult.Error;
                hce.Error.Add(string.Format("{0} - Error connecting to IIS. {1}", StatusResult.Error, ex.Message));
            }

            hru.Errors.Add(hce);
            return(await Task.FromResult <HealthRollup>(hru));
        }