private static void SetOfflineDatabaseInfo(
ServerStore serverStore,
TransactionOperationContext context,
string databaseName,
DatabasesInfo existingDatabasesInfo,
DrivesUsage existingDrivesUsage,
bool disabled)
{
using (var databaseRecord = serverStore.Cluster.ReadRawDatabaseRecord(context, databaseName))
{
if (databaseRecord == null)
{
// database doesn't exist
return;
}
var databaseTopology = databaseRecord.Topology;
var irrelevant = databaseTopology == null ||
databaseTopology.AllNodes.Contains(serverStore.NodeTag) == false;
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
Online = false,
Disabled = disabled,
Irrelevant = irrelevant
};
if (irrelevant == false)
{
// nothing to fetch if irrelevant on this node
UpdateDatabaseInfo(databaseRecord, serverStore, databaseName, existingDrivesUsage, databaseInfoItem);
}
existingDatabasesInfo.Items.Add(databaseInfoItem);
}
}
private static void UpdateMountPoint(
Client.ServerWide.Operations.MountPointUsage mountPointUsage,
string databaseName,
DrivesUsage drivesUsage)
{
var mountPoint = mountPointUsage.DiskSpaceResult.DriveName;
var usage = drivesUsage.Items.FirstOrDefault(x => x.MountPoint == mountPoint);
if (usage == null)
{
usage = new MountPointUsage
{
MountPoint = mountPoint,
VolumeLabel = mountPointUsage.DiskSpaceResult.VolumeLabel,
FreeSpace = mountPointUsage.DiskSpaceResult.TotalFreeSpaceInBytes,
TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSizeInBytes
};
drivesUsage.Items.Add(usage);
}
usage.VolumeLabel = mountPointUsage.DiskSpaceResult.VolumeLabel;
usage.FreeSpace = mountPointUsage.DiskSpaceResult.TotalFreeSpaceInBytes;
usage.TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSizeInBytes;
var existingDatabaseUsage = usage.Items.FirstOrDefault(x => x.Database == databaseName);
if (existingDatabaseUsage == null)
{
existingDatabaseUsage = new DatabaseDiskUsage
{
Database = databaseName
};
usage.Items.Add(existingDatabaseUsage);
}
existingDatabaseUsage.Size += mountPointUsage.UsedSpace;
}
private static void UpdateDatabaseInfo(BlittableJsonReaderObject databaseRecord, ServerStore serverStore, string databaseName, DrivesUsage existingDrivesUsage,
DatabaseInfoItem databaseInfoItem)
{
DatabaseInfo databaseInfo = null;
if (serverStore.DatabaseInfoCache.TryGet(databaseName,
databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)) == false)
{
return;
}
Debug.Assert(databaseInfo != null);
var databaseTopology = serverStore.Cluster.ReadDatabaseTopology(databaseRecord);
databaseRecord.TryGet(nameof(DatabaseRecord.Indexes), out BlittableJsonReaderObject indexes);
var indexesCount = indexes?.Count ?? 0;
databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0;
databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? indexesCount;
databaseInfoItem.ReplicationFactor = databaseTopology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
if (databaseInfo.MountPointsUsage == null)
{
return;
}
foreach (var mountPointUsage in databaseInfo.MountPointsUsage)
{
var driveName = mountPointUsage.DiskSpaceResult.DriveName;
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(
mountPointUsage.DiskSpaceResult.DriveName,
new DriveInfoBase
{
DriveName = driveName
});
if (diskSpaceResult != null)
{
// update the latest drive info
mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult
{
DriveName = diskSpaceResult.DriveName,
VolumeLabel = diskSpaceResult.VolumeLabel,
TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
TotalSizeInBytes = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
};
}
UpdateMountPoint(serverStore.Configuration.Storage, mountPointUsage, databaseName, existingDrivesUsage);
}
}
private static void DiskUsageCheck(DatabaseInfoCache item, DocumentDatabase database, DrivesUsage drivesUsage, CancellationTokenSource cts)
{
foreach (var mountPointUsage in database.GetMountPointsUsage(includeTempBuffers: true))
{
if (cts.IsCancellationRequested)
{
return;
}
UpdateMountPoint(database.Configuration.Storage, mountPointUsage, database.Name, drivesUsage);
item.MountPoints.Add(mountPointUsage);
}
item.NextDiskSpaceCheck = SystemTime.UtcNow.AddSeconds(30);
}
public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, Func <string, bool> isValidFor, CancellationTokenSource cts)
{
var databasesInfo = new DatabasesInfo();
var indexingSpeed = new IndexingSpeed();
var trafficWatch = new TrafficWatch();
var drivesUsage = new DrivesUsage();
trafficWatch.AverageRequestDuration = serverStore.Server.Metrics.Requests.AverageDuration.GetRate();
using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext))
using (transactionContext.OpenReadTransaction())
{
// 1. Fetch databases info
foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue))
{
var databaseName = databaseTuple.ItemName.Substring(Constants.Documents.Prefix.Length);
if (cts.IsCancellationRequested)
{
yield break;
}
if (isValidFor != null && isValidFor(databaseName) == false)
{
continue;
}
if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false)
{
// database does not exist on this server, is offline or disabled
SetOfflineDatabaseInfo(serverStore, transactionContext, databaseName, databasesInfo, drivesUsage, disabled: DatabasesLandlord.IsDatabaseDisabled(databaseTuple.Value));
continue;
}
try
{
var databaseOnline = IsDatabaseOnline(databaseTask, out var database);
if (databaseOnline == false)
{
SetOfflineDatabaseInfo(serverStore, transactionContext, databaseName, databasesInfo, drivesUsage, disabled: false);
continue;
}
var indexingSpeedItem = new IndexingSpeedItem
{
Database = database.Name,
IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.OneSecondRate,
MappedPerSecond = database.Metrics.MapReduceIndexes.MappedPerSec.OneSecondRate,
ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.OneSecondRate
};
indexingSpeed.Items.Add(indexingSpeedItem);
var replicationFactor = GetReplicationFactor(databaseTuple.Value);
var documentsStorage = database.DocumentsStorage;
var indexStorage = database.IndexStore;
var trafficWatchItem = new TrafficWatchItem
{
Database = database.Name,
RequestsPerSecond = (int)database.Metrics.Requests.RequestsPerSec.OneSecondRate,
AverageRequestDuration = database.Metrics.Requests.AverageDuration.GetRate(),
DocumentWritesPerSecond = (int)database.Metrics.Docs.PutsPerSec.OneSecondRate,
AttachmentWritesPerSecond = (int)database.Metrics.Attachments.PutsPerSec.OneSecondRate,
CounterWritesPerSecond = (int)database.Metrics.Counters.PutsPerSec.OneSecondRate,
DocumentsWriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.OneSecondRate,
AttachmentsWriteBytesPerSecond = database.Metrics.Attachments.BytesPutsPerSec.OneSecondRate,
CountersWriteBytesPerSecond = database.Metrics.Counters.BytesPutsPerSec.OneSecondRate
};
trafficWatch.Items.Add(trafficWatchItem);
var currentEnvironmentsHash = database.GetEnvironmentsHash();
if (CachedDatabaseInfo.TryGetValue(database.Name, out var item) && item.Hash == currentEnvironmentsHash)
{
databasesInfo.Items.Add(item.Item);
if (item.NextDiskSpaceCheck < SystemTime.UtcNow)
{
item.MountPoints = new List <Client.ServerWide.Operations.MountPointUsage>();
DiskUsageCheck(item, database, drivesUsage, cts);
}
else
{
foreach (var cachedMountPoint in item.MountPoints)
{
UpdateMountPoint(database.Configuration.Storage, cachedMountPoint, database.Name, drivesUsage);
}
}
}
else
{
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
var databaseInfoItem = new DatabaseInfoItem
{
Database = database.Name,
DocumentsCount = documentsStorage.GetNumberOfDocuments(documentsContext),
IndexesCount = database.IndexStore.Count,
AlertsCount = database.NotificationCenter.GetAlertCount(),
ReplicationFactor = replicationFactor,
ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0),
Online = true
};
databasesInfo.Items.Add(databaseInfoItem);
CachedDatabaseInfo[database.Name] = item = new DatabaseInfoCache
{
Hash = currentEnvironmentsHash,
Item = databaseInfoItem
};
}
DiskUsageCheck(item, database, drivesUsage, cts);
}
}
catch (Exception)
{
SetOfflineDatabaseInfo(serverStore, transactionContext, databaseName, databasesInfo, drivesUsage, disabled: false);
}
}
// 2. Fetch <system> info
if (isValidFor == null)
{
var currentSystemHash = serverStore._env.CurrentReadTransactionId;
var cachedSystemInfoCopy = CachedSystemInfo;
if (currentSystemHash != cachedSystemInfoCopy.Hash || cachedSystemInfoCopy.NextDiskSpaceCheck < SystemTime.UtcNow)
{
var systemInfo = new SystemInfoCache()
{
Hash = currentSystemHash,
NextDiskSpaceCheck = SystemTime.UtcNow.AddSeconds(30),
MountPoints = new List <Client.ServerWide.Operations.MountPointUsage>()
};
// Get new data
var systemEnv = new StorageEnvironmentWithType("<System>", StorageEnvironmentWithType.StorageEnvironmentType.System, serverStore._env);
var systemMountPoints = ServerStore.GetMountPointUsageDetailsFor(systemEnv, includeTempBuffers: true);
foreach (var systemPoint in systemMountPoints)
{
UpdateMountPoint(serverStore.Configuration.Storage, systemPoint, "<System>", drivesUsage);
systemInfo.MountPoints.Add(systemPoint);
}
// Update the cache
Interlocked.Exchange(ref CachedSystemInfo, systemInfo);
}
else
{
// Use existing data
foreach (var systemPoint in cachedSystemInfoCopy.MountPoints)
{
UpdateMountPoint(serverStore.Configuration.Storage, systemPoint, "<System>", drivesUsage);
}
}
}
}
yield return(databasesInfo);
yield return(indexingSpeed);
yield return(trafficWatch);
yield return(drivesUsage);
}
public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, Func <string, bool> isValidFor, CancellationTokenSource cts)
{
var databasesInfo = new DatabasesInfo();
var indexingSpeed = new IndexingSpeed();
var trafficWatch = new TrafficWatch();
var drivesUsage = new DrivesUsage();
using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext))
using (transactionContext.OpenReadTransaction())
{
foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue))
{
var databaseName = databaseTuple.ItemName.Substring(Constants.Documents.Prefix.Length);
if (cts.IsCancellationRequested)
{
yield break;
}
if (isValidFor != null && isValidFor(databaseName) == false)
{
continue;
}
if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false)
{
// database does not exist on this server, is offline or disabled
SetOfflineDatabaseInfo(serverStore, transactionContext, databaseName, databasesInfo, drivesUsage, disabled: IsDatabaseDisabled(databaseTuple.Value));
continue;
}
try
{
var databaseOnline = IsDatabaseOnline(databaseTask, out var database);
if (databaseOnline == false)
{
SetOfflineDatabaseInfo(serverStore, transactionContext, databaseName, databasesInfo, drivesUsage, disabled: false);
continue;
}
var indexingSpeedItem = new IndexingSpeedItem
{
Database = database.Name,
IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.OneSecondRate,
MappedPerSecond = database.Metrics.MapReduceIndexes.MappedPerSec.OneSecondRate,
ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.OneSecondRate
};
indexingSpeed.Items.Add(indexingSpeedItem);
var replicationFactor = GetReplicationFactor(databaseTuple.Value);
var documentsStorage = database.DocumentsStorage;
var indexStorage = database.IndexStore;
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
DocumentsCount = documentsStorage.GetNumberOfDocuments(documentsContext),
IndexesCount = database.IndexStore.Count,
AlertsCount = database.NotificationCenter.GetAlertCount(),
ReplicationFactor = replicationFactor,
ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0),
Online = true
};
databasesInfo.Items.Add(databaseInfoItem);
}
var trafficWatchItem = new TrafficWatchItem
{
Database = databaseName,
RequestsPerSecond = (int)database.Metrics.Requests.RequestsPerSec.OneSecondRate,
WritesPerSecond = (int)database.Metrics.Docs.PutsPerSec.OneSecondRate,
WriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.OneSecondRate
};
trafficWatch.Items.Add(trafficWatchItem);
foreach (var mountPointUsage in database.GetMountPointsUsage())
{
if (cts.IsCancellationRequested)
{
yield break;
}
UpdateMountPoint(mountPointUsage, databaseName, drivesUsage);
}
}
catch (Exception)
{
SetOfflineDatabaseInfo(serverStore, transactionContext, databaseName, databasesInfo, drivesUsage, disabled: false);
}
}
}
yield return(databasesInfo);
yield return(indexingSpeed);
yield return(trafficWatch);
yield return(drivesUsage);
}
private static void UpdateDatabaseInfo(DatabaseRecord databaseRecord, ServerStore serverStore, string databaseName, DrivesUsage existingDrivesUsage,
DatabaseInfoItem databaseInfoItem)
{
DatabaseInfo databaseInfo = null;
if (serverStore.DatabaseInfoCache.TryGet(databaseName,
databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)) == false)
{
return;
}
Debug.Assert(databaseInfo != null);
databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0;
databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
databaseInfoItem.ReplicationFactor = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
if (databaseInfo.MountPointsUsage == null)
{
return;
}
var drives = DriveInfo.GetDrives();
foreach (var mountPointUsage in databaseInfo.MountPointsUsage)
{
var diskSpaceResult = DiskSpaceChecker.GetFreeDiskSpace(mountPointUsage.DiskSpaceResult.DriveName, drives);
if (diskSpaceResult != null)
{
// update the latest drive info
mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult
{
DriveName = diskSpaceResult.DriveName,
VolumeLabel = diskSpaceResult.VolumeLabel,
TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
TotalSizeInBytes = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
};
}
UpdateMountPoint(mountPointUsage, databaseName, existingDrivesUsage);
}
}
public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, CancellationTokenSource cts)
{
var databasesInfo = new DatabasesInfo();
var indexingSpeed = new IndexingSpeed();
var trafficWatch = new TrafficWatch();
var drivesUsage = new DrivesUsage();
using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext))
using (transactionContext.OpenReadTransaction())
{
foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue))
{
var databaseName = databaseTuple.ItemName.Substring(3);
if (cts.IsCancellationRequested)
{
yield break;
}
if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false)
{
// database does not exist in this server or disabled
continue;
}
var databaseOnline = IsDatabaseOnline(databaseTask, out var database);
if (databaseOnline == false)
{
var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _);
if (databaseRecord == null)
{
// database doesn't exist
continue;
}
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
Online = false
};
DatabaseInfo databaseInfo = null;
if (serverStore.DatabaseInfoCache.TryGet(databaseName,
databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)))
{
Debug.Assert(databaseInfo != null);
databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0;
databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
databaseInfoItem.ReplicationFactor = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
}
databasesInfo.Items.Add(databaseInfoItem);
continue;
}
var indexingSpeedItem = new IndexingSpeedItem
{
Database = database.Name,
IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.FiveSecondRate,
MappedPerSecond = database.Metrics.MapReduceIndexes.MappedPerSec.FiveSecondRate,
ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.FiveSecondRate
};
indexingSpeed.Items.Add(indexingSpeedItem);
var replicationFactor = GetReplicationFactor(databaseTuple.Value);
var documentsStorage = database.DocumentsStorage;
var indexStorage = database.IndexStore;
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
DocumentsCount = documentsStorage.GetNumberOfDocuments(documentsContext),
IndexesCount = database.IndexStore.Count,
AlertsCount = database.NotificationCenter.GetAlertCount(),
ReplicationFactor = replicationFactor,
ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0),
Online = true
};
databasesInfo.Items.Add(databaseInfoItem);
}
var trafficWatchItem = new TrafficWatchItem
{
Database = databaseName,
RequestsPerSecond = (int)database.Metrics.Requests.RequestsPerSec.FiveSecondRate,
WritesPerSecond = (int)database.Metrics.Docs.PutsPerSec.FiveSecondRate,
WriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.FiveSecondRate
};
trafficWatch.Items.Add(trafficWatchItem);
foreach (var mountPointUsage in database.GetMountPointsUsage())
{
if (cts.IsCancellationRequested)
{
yield break;
}
var mountPoint = mountPointUsage.DiskSpaceResult.DriveName;
var usage = drivesUsage.Items.FirstOrDefault(x => x.MountPoint == mountPoint);
if (usage == null)
{
usage = new MountPointUsage
{
MountPoint = mountPoint,
VolumeLabel = mountPointUsage.DiskSpaceResult.VolumeLabel,
FreeSpace = mountPointUsage.DiskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
};
drivesUsage.Items.Add(usage);
}
var existingDatabaseUsage = usage.Items.FirstOrDefault(x => x.Database == databaseName);
if (existingDatabaseUsage == null)
{
existingDatabaseUsage = new DatabaseDiskUsage
{
Database = databaseName
};
usage.Items.Add(existingDatabaseUsage);
}
existingDatabaseUsage.Size += mountPointUsage.UsedSpace;
}
}
}
yield return(databasesInfo);
yield return(indexingSpeed);
yield return(trafficWatch);
yield return(drivesUsage);
}