/// <inheritdoc/>
public override async Task ObserveAsync(CancellationToken token)
{
// If set, this observer will only run during the supplied interval.
// See Settings.xml, CertificateObserverConfiguration section, RunInterval parameter for an example...
if (this.RunInterval > TimeSpan.MinValue &&
DateTime.Now.Subtract(this.LastRunDateTime) < this.RunInterval)
{
return;
}
this.SetErrorWarningThresholds();
DriveInfo[] allDrives = DriveInfo.GetDrives();
var diskUsage = new DiskUsage();
if (ObserverManager.ObserverWebAppDeployed)
{
this.diskInfo = new StringBuilder();
}
try
{
int readyCount = 0;
foreach (var d in allDrives)
{
token.ThrowIfCancellationRequested();
if (d.IsReady)
{
readyCount++;
// This section only needs to run if you have the FabricObserverWebApi app installed...
if (ObserverManager.ObserverWebAppDeployed)
{
this.diskInfo.AppendFormat("\n\nDrive Name: {0}\n", d.Name);
// Logging...
this.diskInfo.AppendFormat("Drive Type: {0}\n", d.DriveType);
this.diskInfo.AppendFormat(" Volume Label : {0}\n", d.VolumeLabel);
this.diskInfo.AppendFormat(" Filesystem : {0}\n", d.DriveFormat);
this.diskInfo.AppendFormat(" Total Disk Size: {0} GB\n", d.TotalSize / 1024 / 1024 / 1024);
this.diskInfo.AppendFormat(" Root Directory : {0}\n", d.RootDirectory);
this.diskInfo.AppendFormat(" Free User : {0} GB\n", d.AvailableFreeSpace / 1024 / 1024 / 1024);
this.diskInfo.AppendFormat(" Free Total: {0} GB\n", d.TotalFreeSpace / 1024 / 1024 / 1024);
this.diskInfo.AppendFormat(" % Used : {0}%\n", diskUsage.GetCurrentDiskSpaceUsedPercent(d.Name));
}
// Setup monitoring data structures...
string id = d.Name.Substring(0, 1);
// Since these live across iterations, do not duplicate them in the containing list...
// Disk space %...
if (!this.diskSpacePercentageUsageData.Any(data => data.Id == id))
{
this.diskSpacePercentageUsageData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceUsagePercentage, id));
}
if (!this.diskSpaceUsageData.Any(data => data.Id == id))
{
this.diskSpaceUsageData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceUsageMB, id));
}
if (!this.diskSpaceAvailableData.Any(data => data.Id == id))
{
this.diskSpaceAvailableData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceAvailableMB, id));
}
if (!this.diskSpaceTotalData.Any(data => data.Id == id))
{
this.diskSpaceTotalData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceTotalMB, id));
}
// Current disk queue length...
if (!this.diskAverageQueueLengthData.Any(data => data.Id == id))
{
this.diskAverageQueueLengthData.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.DiskAverageQueueLength, id));
}
// Generate data over time (_monitorDuration...) for use in ReportAsync health analysis...
this.stopWatch?.Start();
while (this.stopWatch?.Elapsed <= this.monitorDuration)
{
token.ThrowIfCancellationRequested();
this.diskSpacePercentageUsageData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetCurrentDiskSpaceUsedPercent(id));
this.diskSpaceUsageData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetUsedDiskSpace(id, SizeUnit.Megabytes));
this.diskSpaceAvailableData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetAvailabeDiskSpace(id, SizeUnit.Megabytes));
this.diskSpaceTotalData.FirstOrDefault(
x => x.Id == id)
.Data.Add(DiskUsage.GetTotalDiskSpace(id, SizeUnit.Megabytes));
this.diskAverageQueueLengthData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2)));
Thread.Sleep(250);
}
// This section only needs to run if you have the FabricObserverWebApi app installed...
if (ObserverManager.ObserverWebAppDeployed)
{
this.diskInfo.AppendFormat(
"{0}",
this.GetWindowsPerfCounterDetailsText(
this.diskAverageQueueLengthData.FirstOrDefault(
x => x.Id == d.Name.Substring(0, 1)).Data,
"Avg. Disk Queue Length"));
}
this.stopWatch.Stop();
this.stopWatch.Reset();
}
}
}
finally
{
diskUsage.Dispose();
diskUsage = null;
}
await this.ReportAsync(token).ConfigureAwait(true);
this.LastRunDateTime = DateTime.Now;
}
public override async Task ObserveAsync(CancellationToken token)
{
// If set, this observer will only run during the supplied interval.
// See Settings.xml, CertificateObserverConfiguration section, RunInterval parameter for an example.
if (RunInterval > TimeSpan.MinValue &&
DateTime.Now.Subtract(LastRunDateTime) < RunInterval)
{
return;
}
SetErrorWarningThresholds();
DriveInfo[] allDrives = DriveInfo.GetDrives();
if (ObserverManager.ObserverWebAppDeployed)
{
this.diskInfo = new StringBuilder();
}
foreach (var d in allDrives)
{
token.ThrowIfCancellationRequested();
if (!DiskUsage.ShouldCheckDrive(d))
{
continue;
}
// This section only needs to run if you have the FabricObserverWebApi app installed.
if (ObserverManager.ObserverWebAppDeployed)
{
_ = this.diskInfo.AppendFormat("\n\nDrive Name: {0}\n", d.Name);
// Logging.
_ = this.diskInfo.AppendFormat("Drive Type: {0}\n", d.DriveType);
_ = this.diskInfo.AppendFormat(" Volume Label : {0}\n", d.VolumeLabel);
_ = this.diskInfo.AppendFormat(" Filesystem : {0}\n", d.DriveFormat);
_ = this.diskInfo.AppendFormat(" Total Disk Size: {0} GB\n", d.TotalSize / 1024 / 1024 / 1024);
_ = this.diskInfo.AppendFormat(" Root Directory : {0}\n", d.RootDirectory);
_ = this.diskInfo.AppendFormat(" Free User : {0} GB\n", d.AvailableFreeSpace / 1024 / 1024 / 1024);
_ = this.diskInfo.AppendFormat(" Free Total: {0} GB\n", d.TotalFreeSpace / 1024 / 1024 / 1024);
_ = this.diskInfo.AppendFormat(" % Used : {0}%\n", DiskUsage.GetCurrentDiskSpaceUsedPercent(d.Name));
}
// Setup monitoring data structures.
string id = d.Name;
// Since these live across iterations, do not duplicate them in the containing list.
// Disk space %.
if (this.DiskSpaceUsagePercentageData.All(data => data.Id != id) && (this.DiskSpacePercentErrorThreshold > 0 || this.DiskSpacePercentWarningThreshold > 0))
{
this.DiskSpaceUsagePercentageData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceUsagePercentage, id, DataCapacity));
}
// Current disk queue length.
if (this.DiskAverageQueueLengthData.All(data => data.Id != id) && (this.AverageQueueLengthErrorThreshold > 0 || this.AverageQueueLengthWarningThreshold > 0))
{
this.DiskAverageQueueLengthData.Add(new FabricResourceUsageData <float>(ErrorWarningProperty.DiskAverageQueueLength, id, DataCapacity));
}
// This data is just used for Telemetry today.
if (this.DiskSpaceAvailableMbData.All(data => data.Id != id))
{
this.DiskSpaceAvailableMbData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceAvailableMb, id, DataCapacity));
}
// This data is just used for Telemetry today.
if (this.DiskSpaceTotalMbData.All(data => data.Id != id))
{
this.DiskSpaceTotalMbData.Add(new FabricResourceUsageData <double>(ErrorWarningProperty.DiskSpaceTotalMb, id, DataCapacity));
}
// It is important to check if code is running on Windows, since d.Name.Substring(0, 2) will fail on Linux for / (root) mount point.
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (this.AverageQueueLengthErrorThreshold > 0 || this.AverageQueueLengthWarningThreshold > 0)
{
// Warm up counter.
_ = DiskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2));
this.DiskAverageQueueLengthData.Single(x => x.Id == id).Data.Add(DiskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2)));
}
}
if (this.DiskSpacePercentErrorThreshold > 0 || this.DiskSpacePercentWarningThreshold > 0)
{
this.DiskSpaceUsagePercentageData.Single(x => x.Id == id).Data.Add(DiskUsage.GetCurrentDiskSpaceUsedPercent(id));
}
this.DiskSpaceAvailableMbData.Single(x => x.Id == id).Data.Add(DiskUsage.GetAvailableDiskSpace(id, SizeUnit.Megabytes));
this.DiskSpaceTotalMbData.Single(x => x.Id == id).Data.Add(DiskUsage.GetTotalDiskSpace(id, SizeUnit.Megabytes));
// This section only needs to run if you have the FabricObserverWebApi app installed.
if (ObserverManager.ObserverWebAppDeployed && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_ = this.diskInfo.AppendFormat(
"{0}",
GetWindowsPerfCounterDetailsText(
this.DiskAverageQueueLengthData.FirstOrDefault(
x => x.Id == d.Name.Substring(0, 1))
?.Data,
"Avg. Disk Queue Length"));
}
RunDuration = this.stopWatch.Elapsed;
}
await ReportAsync(token).ConfigureAwait(true);
LastRunDateTime = DateTime.Now;
}
// Windows process dmp creator.
public bool DumpServiceProcess(int processId, DumpType dumpType = DumpType.Full)
{
if (string.IsNullOrEmpty(this.dumpsPath))
{
return(false);
}
string processName = string.Empty;
NativeMethods.MINIDUMP_TYPE miniDumpType;
switch (dumpType)
{
case DumpType.Full:
miniDumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemory |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemoryInfo |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithHandleData |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithThreadInfo |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithUnloadedModules;
break;
case DumpType.MiniPlus:
miniDumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpWithPrivateReadWriteMemory |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithDataSegs |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithHandleData |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithFullMemoryInfo |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithThreadInfo |
NativeMethods.MINIDUMP_TYPE.MiniDumpWithUnloadedModules;
break;
case DumpType.Mini:
miniDumpType = NativeMethods.MINIDUMP_TYPE.MiniDumpWithIndirectlyReferencedMemory |
NativeMethods.MINIDUMP_TYPE.MiniDumpScanMemory;
break;
default:
throw new ArgumentOutOfRangeException(nameof(dumpType), dumpType, null);
}
try
{
// This is to ensure friendly-name of resulting dmp file.
processName = Process.GetProcessById(processId).ProcessName;
if (string.IsNullOrEmpty(processName))
{
return(false);
}
IntPtr processHandle = Process.GetProcessById(processId).Handle;
processName += "_" + DateTime.Now.ToString("ddMMyyyyHHmmss") + ".dmp";
// Check disk space availability before writing dump file.
// This will not work on Linux
string driveName = this.dumpsPath.Substring(0, 2);
if (DiskUsage.GetCurrentDiskSpaceUsedPercent(driveName) > 90)
{
HealthReporter.ReportFabricObserverServiceHealth(
FabricServiceContext.ServiceName.OriginalString,
ObserverName,
HealthState.Warning,
"Not enough disk space available for dump file creation.");
return(false);
}
using (var file = File.Create(Path.Combine(this.dumpsPath, processName)))
{
if (!NativeMethods.MiniDumpWriteDump(
processHandle,
(uint)processId,
file.SafeFileHandle,
miniDumpType,
IntPtr.Zero,
IntPtr.Zero,
IntPtr.Zero))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
}
return(true);
}
catch (Exception e) when(e is ArgumentException || e is InvalidOperationException || e is Win32Exception)
{
ObserverLogger.LogInfo(
$"Unable to generate dump file {processName} with error{Environment.NewLine}{e}");
}
return(false);
}
/// <inheritdoc/>
public override async Task ObserveAsync(CancellationToken token)
{
this.SetErrorWarningThresholds();
DriveInfo[] allDrives = DriveInfo.GetDrives();
var diskUsage = new DiskUsage();
this.diskInfo = new StringBuilder();
try
{
int readyCount = 0;
foreach (var d in allDrives)
{
token.ThrowIfCancellationRequested();
if (d.IsReady)
{
readyCount++;
// This section only needs to run if you have the FabricObserverWebApi app installed...
// Always log since these are the identifiers of each detected drive...
this.diskInfo.AppendFormat("\n\nDrive Name: {0}\n", d.Name);
// Logging...
this.diskInfo.AppendFormat("Drive Type: {0}\n", d.DriveType);
this.diskInfo.AppendFormat(" Volume Label : {0}\n", d.VolumeLabel);
this.diskInfo.AppendFormat(" Filesystem : {0}\n", d.DriveFormat);
this.diskInfo.AppendFormat(" Total Disk Size: {0} GB\n", d.TotalSize / 1024 / 1024 / 1024);
this.diskInfo.AppendFormat(" Root Directory : {0}\n", d.RootDirectory);
this.diskInfo.AppendFormat(" Free User : {0} GB\n", d.AvailableFreeSpace / 1024 / 1024 / 1024);
this.diskInfo.AppendFormat(" Free Total: {0} GB\n", d.TotalFreeSpace / 1024 / 1024 / 1024);
this.diskInfo.AppendFormat(" % Used : {0}%\n", diskUsage.GetCurrentDiskSpaceUsedPercent(d.Name));
// Setup monitoring data structures...
string id = d.Name.Substring(0, 1);
// Since these live across iterations, do not duplicate them in the containing list...
if (!this.diskIOReadsData.Any(data => data.Id == id))
{
this.diskIOReadsData.Add(new FabricResourceUsageData <float>("Disk sec/Read (ms)", id));
}
if (!this.diskIOWritesData.Any(data => data.Id == id))
{
this.diskIOWritesData.Add(new FabricResourceUsageData <float>("Disk sec/Write (ms)", id));
}
// Disk space %...
if (!this.diskSpacePercentageUsageData.Any(data => data.Id == id))
{
this.diskSpacePercentageUsageData.Add(new FabricResourceUsageData <double>("Disk Space Consumption %", id));
}
if (!this.diskSpaceUsageData.Any(data => data.Id == id))
{
this.diskSpaceUsageData.Add(new FabricResourceUsageData <double>("Disk Space Consumption MB", id));
}
if (!this.diskSpaceAvailableData.Any(data => data.Id == id))
{
this.diskSpaceAvailableData.Add(new FabricResourceUsageData <double>("Disk Space Available MB", id));
}
if (!this.diskSpaceTotalData.Any(data => data.Id == id))
{
this.diskSpaceTotalData.Add(new FabricResourceUsageData <double>("Disk Space Total MB", id));
}
// Current disk queue length...
if (!this.diskAverageQueueLengthData.Any(data => data.Id == id))
{
this.diskAverageQueueLengthData.Add(new FabricResourceUsageData <float>("Average Disk Queue Length", id));
}
// Generate data over time (_monitorDuration...) for use in ReportAsync health analysis...
this.stopWatch?.Start();
while (this.stopWatch?.Elapsed <= this.monitorDuration)
{
token.ThrowIfCancellationRequested();
this.diskIOReadsData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.PerfCounterGetDiskIOInfo(d.Name.Substring(0, 2), "LogicalDisk", "Avg. Disk sec/Read") * 1000);
this.diskIOWritesData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.PerfCounterGetDiskIOInfo(d.Name.Substring(0, 2), "LogicalDisk", "Avg. Disk sec/Write") * 1000);
this.diskSpacePercentageUsageData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetCurrentDiskSpaceUsedPercent(id));
this.diskSpaceUsageData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetUsedDiskSpace(id, SizeUnit.Megabytes));
this.diskSpaceAvailableData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetAvailabeDiskSpace(id, SizeUnit.Megabytes));
this.diskSpaceTotalData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetTotalDiskSpace(id, SizeUnit.Megabytes));
this.diskAverageQueueLengthData.FirstOrDefault(
x => x.Id == id)
.Data.Add(diskUsage.GetAverageDiskQueueLength(d.Name.Substring(0, 2)));
Thread.Sleep(250);
}
// This section only needs to run if you have the FabricObserverWebApi app installed...
this.diskInfo.AppendFormat(
"{0}",
this.GetWindowsPerfCounterDetailsText(
this.diskIOReadsData.FirstOrDefault(
x => x.Id == d.Name.Substring(0, 1)).Data,
"Avg. Disk sec/Read"));
this.diskInfo.AppendFormat(
"{0}",
this.GetWindowsPerfCounterDetailsText(
this.diskIOWritesData.FirstOrDefault(
x => x.Id == d.Name.Substring(0, 1)).Data,
"Avg. Disk sec/Write"));
this.diskInfo.AppendFormat(
"{0}",
this.GetWindowsPerfCounterDetailsText(
this.diskAverageQueueLengthData.FirstOrDefault(
x => x.Id == d.Name.Substring(0, 1)).Data,
"Avg. Disk Queue Length"));
this.stopWatch.Stop();
this.stopWatch.Reset();
}
}
}
finally
{
diskUsage.Dispose();
diskUsage = null;
}
await this.ReportAsync(token).ConfigureAwait(true);
this.LastRunDateTime = DateTime.Now;
}