Example #1
0
        public override async Task ObserveAsync(CancellationToken token)
        {
            // If set, this observer will only run during the supplied interval.
            // See Settings.xml, CertificateObserverConfiguration section, RunInterval parameter for an example.
            if (RunInterval > TimeSpan.MinValue &&
                DateTime.Now.Subtract(LastRunDateTime) < RunInterval)
            {
                return;
            }

            SetErrorWarningThresholds();

            DriveInfo[] allDrives = DriveInfo.GetDrives();

            if (ObserverManager.ObserverWebAppDeployed)
            {
                this.diskInfo = new StringBuilder();
            }

            foreach (var d in allDrives)
            {
                token.ThrowIfCancellationRequested();

                if (!DiskUsage.ShouldCheckDrive(d))
                {
                    continue;
                }

                // This section only needs to run if you have the FabricObserverWebApi app installed.
                if (ObserverManager.ObserverWebAppDeployed)
                {
                    _ = this.diskInfo.AppendFormat("\n\nDrive Name: {0}\n", d.Name);

                    // Logging.
                    _ = this.diskInfo.AppendFormat("Drive Type: {0}\n", d.DriveType);
                    _ = this.diskInfo.AppendFormat("  Volume Label   : {0}\n", d.VolumeLabel);
                    _ = this.diskInfo.AppendFormat("  Filesystem     : {0}\n", d.DriveFormat);
                    _ = this.diskInfo.AppendFormat("  Total Disk Size: {0} GB\n", d.TotalSize / 1024 / 1024 / 1024);
                    _ = this.diskInfo.AppendFormat("  Root Directory : {0}\n", d.RootDirectory);
                    _ = this.diskInfo.AppendFormat("  Free User : {0} GB\n", d.AvailableFreeSpace / 1024 / 1024 / 1024);
                    _ = this.diskInfo.AppendFormat("  Free Total: {0} GB\n", d.TotalFreeSpace / 1024 / 1024 / 1024);
                    _ = this.diskInfo.AppendFormat("  % Used    : {0}%\n", DiskUsage.GetCurrentDiskSpaceUsedPercent(d.Name));
                }

                // Setup monitoring data structures.
                string id = d.Name;

                // Since these live across iterations, do not duplicate them in the containing list.
                // Disk space %.
                if (this.DiskSpaceUsagePercentageData.All(data => data.Id != id) && (this.DiskSpacePercentErrorThreshold > 0 || this.DiskSpacePercentWarningThreshold > 0))
                {
                    this.DiskSpaceUsagePercentageData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceUsagePercentage, id, DataCapacity));
                }

                // Current disk queue length.
                if (this.DiskAverageQueueLengthData.All(data => data.Id != id) && (this.AverageQueueLengthErrorThreshold > 0 || this.AverageQueueLengthWarningThreshold > 0))
                {
                    this.DiskAverageQueueLengthData.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.DiskAverageQueueLength, id, DataCapacity));
                }

                // This data is just used for Telemetry today.
                if (this.DiskSpaceAvailableMbData.All(data => data.Id != id))
                {
                    this.DiskSpaceAvailableMbData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceAvailableMb, id, DataCapacity));
                }

                // This data is just used for Telemetry today.
                if (this.DiskSpaceTotalMbData.All(data => data.Id != id))
                {
                    this.DiskSpaceTotalMbData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceTotalMb, id, DataCapacity));
                }

                // It is important to check if code is running on Windows, since d.Name.Substring(0, 2) will fail on Linux for / (root) mount point.
                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                {
                    if (this.AverageQueueLengthErrorThreshold > 0 || this.AverageQueueLengthWarningThreshold > 0)
                    {
                        // Warm up counter.
                        _ = DiskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2));

                        this.DiskAverageQueueLengthData.Single(x => x.Id == id).Data.Add(DiskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2)));
                    }
                }

                if (this.DiskSpacePercentErrorThreshold > 0 || this.DiskSpacePercentWarningThreshold > 0)
                {
                    this.DiskSpaceUsagePercentageData.Single(x => x.Id == id).Data.Add(DiskUsage.GetCurrentDiskSpaceUsedPercent(id));
                }

                this.DiskSpaceAvailableMbData.Single(x => x.Id == id).Data.Add(DiskUsage.GetAvailableDiskSpace(id, SizeUnit.Megabytes));
                this.DiskSpaceTotalMbData.Single(x => x.Id == id).Data.Add(DiskUsage.GetTotalDiskSpace(id, SizeUnit.Megabytes));

                // This section only needs to run if you have the FabricObserverWebApi app installed.
                if (ObserverManager.ObserverWebAppDeployed && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                {
                    _ = this.diskInfo.AppendFormat(
                        "{0}",
                        GetWindowsPerfCounterDetailsText(
                            this.DiskAverageQueueLengthData.FirstOrDefault(
                                x => x.Id == d.Name.Substring(0, 1))
                            ?.Data,
                            "Avg. Disk Queue Length"));
                }

                RunDuration = this.stopWatch.Elapsed;
            }

            await ReportAsync(token).ConfigureAwait(true);

            LastRunDateTime = DateTime.Now;
        }
Example #2
0
        /// <inheritdoc/>
        public override async Task ObserveAsync(CancellationToken token)
        {
            // If set, this observer will only run during the supplied interval.
            // See Settings.xml, CertificateObserverConfiguration section, RunInterval parameter for an example...
            if (this.RunInterval > TimeSpan.MinValue &&
                DateTime.Now.Subtract(this.LastRunDateTime) < this.RunInterval)
            {
                return;
            }

            this.SetErrorWarningThresholds();

            DriveInfo[] allDrives = DriveInfo.GetDrives();
            var         diskUsage = new DiskUsage();

            if (ObserverManager.ObserverWebAppDeployed)
            {
                this.diskInfo = new StringBuilder();
            }

            try
            {
                int readyCount = 0;
                foreach (var d in allDrives)
                {
                    token.ThrowIfCancellationRequested();

                    if (d.IsReady)
                    {
                        readyCount++;

                        // This section only needs to run if you have the FabricObserverWebApi app installed...
                        if (ObserverManager.ObserverWebAppDeployed)
                        {
                            this.diskInfo.AppendFormat("\n\nDrive Name: {0}\n", d.Name);

                            // Logging...
                            this.diskInfo.AppendFormat("Drive Type: {0}\n", d.DriveType);
                            this.diskInfo.AppendFormat("  Volume Label   : {0}\n", d.VolumeLabel);
                            this.diskInfo.AppendFormat("  Filesystem     : {0}\n", d.DriveFormat);
                            this.diskInfo.AppendFormat("  Total Disk Size: {0} GB\n", d.TotalSize / 1024 / 1024 / 1024);
                            this.diskInfo.AppendFormat("  Root Directory : {0}\n", d.RootDirectory);
                            this.diskInfo.AppendFormat("  Free User : {0} GB\n", d.AvailableFreeSpace / 1024 / 1024 / 1024);
                            this.diskInfo.AppendFormat("  Free Total: {0} GB\n", d.TotalFreeSpace / 1024 / 1024 / 1024);
                            this.diskInfo.AppendFormat("  % Used    : {0}%\n", diskUsage.GetCurrentDiskSpaceUsedPercent(d.Name));
                        }

                        // Setup monitoring data structures...
                        string id = d.Name.Substring(0, 1);

                        // Since these live across iterations, do not duplicate them in the containing list...
                        // Disk space %...
                        if (!this.diskSpacePercentageUsageData.Any(data => data.Id == id))
                        {
                            this.diskSpacePercentageUsageData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceUsagePercentage, id));
                        }

                        if (!this.diskSpaceUsageData.Any(data => data.Id == id))
                        {
                            this.diskSpaceUsageData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceUsageMB, id));
                        }

                        if (!this.diskSpaceAvailableData.Any(data => data.Id == id))
                        {
                            this.diskSpaceAvailableData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceAvailableMB, id));
                        }

                        if (!this.diskSpaceTotalData.Any(data => data.Id == id))
                        {
                            this.diskSpaceTotalData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceTotalMB, id));
                        }

                        // Current disk queue length...
                        if (!this.diskAverageQueueLengthData.Any(data => data.Id == id))
                        {
                            this.diskAverageQueueLengthData.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.DiskAverageQueueLength, id));
                        }

                        // Generate data over time (_monitorDuration...) for use in ReportAsync health analysis...
                        this.stopWatch?.Start();

                        while (this.stopWatch?.Elapsed <= this.monitorDuration)
                        {
                            token.ThrowIfCancellationRequested();

                            this.diskSpacePercentageUsageData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetCurrentDiskSpaceUsedPercent(id));

                            this.diskSpaceUsageData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetUsedDiskSpace(id, SizeUnit.Megabytes));

                            this.diskSpaceAvailableData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetAvailabeDiskSpace(id, SizeUnit.Megabytes));

                            this.diskSpaceTotalData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(DiskUsage.GetTotalDiskSpace(id, SizeUnit.Megabytes));

                            this.diskAverageQueueLengthData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2)));

                            Thread.Sleep(250);
                        }

                        // This section only needs to run if you have the FabricObserverWebApi app installed...
                        if (ObserverManager.ObserverWebAppDeployed)
                        {
                            this.diskInfo.AppendFormat(
                                "{0}",
                                this.GetWindowsPerfCounterDetailsText(
                                    this.diskAverageQueueLengthData.FirstOrDefault(
                                        x => x.Id == d.Name.Substring(0, 1)).Data,
                                    "Avg. Disk Queue Length"));
                        }

                        this.stopWatch.Stop();
                        this.stopWatch.Reset();
                    }
                }
            }
            finally
            {
                diskUsage.Dispose();
                diskUsage = null;
            }

            await this.ReportAsync(token).ConfigureAwait(true);

            this.LastRunDateTime = DateTime.Now;
        }
        /// <inheritdoc/>
        public override async Task ObserveAsync(CancellationToken token)
        {
            this.SetErrorWarningThresholds();

            DriveInfo[] allDrives = DriveInfo.GetDrives();
            var         diskUsage = new DiskUsage();

            this.diskInfo = new StringBuilder();

            try
            {
                int readyCount = 0;
                foreach (var d in allDrives)
                {
                    token.ThrowIfCancellationRequested();

                    if (d.IsReady)
                    {
                        readyCount++;

                        // This section only needs to run if you have the FabricObserverWebApi app installed...
                        // Always log since these are the identifiers of each detected drive...
                        this.diskInfo.AppendFormat("\n\nDrive Name: {0}\n", d.Name);

                        // Logging...
                        this.diskInfo.AppendFormat("Drive Type: {0}\n", d.DriveType);
                        this.diskInfo.AppendFormat("  Volume Label   : {0}\n", d.VolumeLabel);
                        this.diskInfo.AppendFormat("  Filesystem     : {0}\n", d.DriveFormat);
                        this.diskInfo.AppendFormat("  Total Disk Size: {0} GB\n", d.TotalSize / 1024 / 1024 / 1024);
                        this.diskInfo.AppendFormat("  Root Directory : {0}\n", d.RootDirectory);
                        this.diskInfo.AppendFormat("  Free User : {0} GB\n", d.AvailableFreeSpace / 1024 / 1024 / 1024);
                        this.diskInfo.AppendFormat("  Free Total: {0} GB\n", d.TotalFreeSpace / 1024 / 1024 / 1024);
                        this.diskInfo.AppendFormat("  % Used    : {0}%\n", diskUsage.GetCurrentDiskSpaceUsedPercent(d.Name));

                        // Setup monitoring data structures...
                        string id = d.Name.Substring(0, 1);

                        // Since these live across iterations, do not duplicate them in the containing list...
                        if (!this.diskIOReadsData.Any(data => data.Id == id))
                        {
                            this.diskIOReadsData.Add(new FabricResourceUsageData <float>("Disk sec/Read (ms)", id));
                        }

                        if (!this.diskIOWritesData.Any(data => data.Id == id))
                        {
                            this.diskIOWritesData.Add(new FabricResourceUsageData <float>("Disk sec/Write (ms)", id));
                        }

                        // Disk space %...
                        if (!this.diskSpacePercentageUsageData.Any(data => data.Id == id))
                        {
                            this.diskSpacePercentageUsageData.Add(new FabricResourceUsageData <double>("Disk Space Consumption %", id));
                        }

                        if (!this.diskSpaceUsageData.Any(data => data.Id == id))
                        {
                            this.diskSpaceUsageData.Add(new FabricResourceUsageData <double>("Disk Space Consumption MB", id));
                        }

                        if (!this.diskSpaceAvailableData.Any(data => data.Id == id))
                        {
                            this.diskSpaceAvailableData.Add(new FabricResourceUsageData <double>("Disk Space Available MB", id));
                        }

                        if (!this.diskSpaceTotalData.Any(data => data.Id == id))
                        {
                            this.diskSpaceTotalData.Add(new FabricResourceUsageData <double>("Disk Space Total MB", id));
                        }

                        // Current disk queue length...
                        if (!this.diskAverageQueueLengthData.Any(data => data.Id == id))
                        {
                            this.diskAverageQueueLengthData.Add(new FabricResourceUsageData <float>("Average Disk Queue Length", id));
                        }

                        // Generate data over time (_monitorDuration...) for use in ReportAsync health analysis...
                        this.stopWatch?.Start();

                        while (this.stopWatch?.Elapsed <= this.monitorDuration)
                        {
                            token.ThrowIfCancellationRequested();

                            this.diskIOReadsData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.PerfCounterGetDiskIOInfo(d.Name.Substring(0, 2), "LogicalDisk", "Avg. Disk sec/Read") * 1000);

                            this.diskIOWritesData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.PerfCounterGetDiskIOInfo(d.Name.Substring(0, 2), "LogicalDisk", "Avg. Disk sec/Write") * 1000);

                            this.diskSpacePercentageUsageData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetCurrentDiskSpaceUsedPercent(id));

                            this.diskSpaceUsageData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetUsedDiskSpace(id, SizeUnit.Megabytes));

                            this.diskSpaceAvailableData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetAvailabeDiskSpace(id, SizeUnit.Megabytes));

                            this.diskSpaceTotalData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetTotalDiskSpace(id, SizeUnit.Megabytes));

                            this.diskAverageQueueLengthData.FirstOrDefault(
                                x => x.Id == id)
                            .Data.Add(diskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2)));

                            Thread.Sleep(250);
                        }

                        // This section only needs to run if you have the FabricObserverWebApi app installed...
                        this.diskInfo.AppendFormat(
                            "{0}",
                            this.GetWindowsPerfCounterDetailsText(
                                this.diskIOReadsData.FirstOrDefault(
                                    x => x.Id == d.Name.Substring(0, 1)).Data,
                                "Avg. Disk sec/Read"));
                        this.diskInfo.AppendFormat(
                            "{0}",
                            this.GetWindowsPerfCounterDetailsText(
                                this.diskIOWritesData.FirstOrDefault(
                                    x => x.Id == d.Name.Substring(0, 1)).Data,
                                "Avg. Disk sec/Write"));

                        this.diskInfo.AppendFormat(
                            "{0}",
                            this.GetWindowsPerfCounterDetailsText(
                                this.diskAverageQueueLengthData.FirstOrDefault(
                                    x => x.Id == d.Name.Substring(0, 1)).Data,
                                "Avg. Disk Queue Length"));

                        this.stopWatch.Stop();
                        this.stopWatch.Reset();
                    }
                }
            }
            finally
            {
                diskUsage.Dispose();
                diskUsage = null;
            }

            await this.ReportAsync(token).ConfigureAwait(true);

            this.LastRunDateTime = DateTime.Now;
        }