public static void SetFileLength(SafeFileHandle fileHandle, long length, string filePath)
{
if (SetFilePointerEx(fileHandle, length, IntPtr.Zero, Win32NativeFileMoveMethod.Begin) == false)
{
var exception = new Win32Exception(Marshal.GetLastWin32Error());
throw new IOException($"Could not move the pointer of file {filePath}", exception);
}
if (SetEndOfFile(fileHandle) == false)
{
var lastError = Marshal.GetLastWin32Error();
if (lastError == (int)Win32NativeFileErrors.ERROR_DISK_FULL)
{
var directoryPath = Path.GetDirectoryName(filePath);
// disk space info is expecting the directory path and not the file path
var driveInfo = DiskSpaceChecker.GetDiskSpaceInfo(directoryPath);
throw new DiskFullException(filePath, length, driveInfo?.TotalFreeSpace.GetValue(SizeUnit.Bytes), new Win32Exception(lastError).Message);
}
var exception = new Win32Exception(lastError);
if (lastError == (int)Win32NativeFileErrors.ERROR_NOT_READY ||
lastError == (int)Win32NativeFileErrors.ERROR_FILE_NOT_FOUND)
{
throw new IOException($"Could not set the size of file {filePath} because it is inaccessible.", exception);
}
throw new IOException($"Could not set the size of file {filePath} to {Sizes.Humane(length)}", exception);
}
}
public static void TryThrowingBetterException(SystemException e, LuceneVoronDirectory directory)
{
if (e.Message.StartsWith("this writer hit an OutOfMemoryError"))
{
ThrowOutOfMemoryException();
}
if (e is Win32Exception win32Exception && win32Exception.IsOutOfMemory())
{
ThrowOutOfMemoryException();
}
if (e.InnerException is VoronUnrecoverableErrorException)
{
VoronUnrecoverableErrorException.Raise("Index data is corrupted", e);
}
if (e.IsOutOfDiskSpaceException())
{
// this commit stage is written to the temp scratch buffers
var fullPath = directory.TempFullPath;
var driveInfo = DiskSpaceChecker.GetDiskSpaceInfo(fullPath);
var freeSpace = driveInfo != null?driveInfo.TotalFreeSpace.ToString() : "N/A";
throw new DiskFullException($"There isn't enough space to commit the index to {fullPath}. " +
$"Currently available space: {freeSpace}", e);
}
void ThrowOutOfMemoryException()
{
throw new OutOfMemoryException("Index writer hit OOM during commit", e);
}
}
private void ThrowDiskFullException()
{
var folderPath = _fileCache.FullPath;
var driveInfo = DiskSpaceChecker.GetDiskSpaceInfo(folderPath);
var freeSpace = driveInfo != null?driveInfo.TotalFreeSpace.ToString() : "N/A";
throw new DiskFullException($"There isn't enough space to flush the buffer in: {folderPath}. " +
$"Currently available space: {freeSpace}");
}
private void ThrowDiskFullException()
{
var directory = Path.GetDirectoryName(_fileTempPath);
var driveInfo = DiskSpaceChecker.GetDiskSpaceInfo(directory);
var freeSpace = driveInfo != null?driveInfo.TotalFreeSpace.ToString() : "N/A";
throw new DiskFullException($"There isn't enough space to flush the buffer of the file: {_fileTempPath}. " +
$"Currently available space: {freeSpace}");
}
public async Task UpdateDirectoryResult(string databaseName, string error)
{
var drivesInfo = PlatformDetails.RunningOnPosix ? DriveInfo.GetDrives() : null;
var driveInfo = DiskSpaceChecker.GetDriveInfo(_path, drivesInfo, out var realPath);
var diskSpaceInfo = DiskSpaceChecker.GetDiskSpaceInfo(driveInfo.DriveName);
if (CanAccessPath(_path, out var pathAccessError) == false)
{
error = pathAccessError;
}
var currentNodeInfo = new SingleNodeDataDirectoryResult
{
NodeTag = _serverStore.NodeTag,
FullPath = realPath,
FreeSpaceInBytes = diskSpaceInfo?.TotalFreeSpace.GetValue(SizeUnit.Bytes) ?? 0,
FreeSpaceHumane = diskSpaceInfo?.TotalFreeSpace.ToString(),
TotalSpaceInBytes = diskSpaceInfo?.TotalSize.GetValue(SizeUnit.Bytes) ?? 0,
TotalSpaceHumane = diskSpaceInfo?.TotalSize.ToString(),
Error = error
};
if (_getNodesInfo == false)
{
// write info of a single node
using (_serverStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, _responseBodyStream))
{
context.Write(writer, currentNodeInfo.ToJson());
}
return;
}
var clusterTopology = _serverStore.GetClusterTopology();
var relevantNodes = GetRelevantNodes(databaseName, clusterTopology);
var dataDirectoryResult = new DataDirectoryResult();
dataDirectoryResult.List.Add(currentNodeInfo);
if (relevantNodes.Count > 1)
{
await UpdateNodesDirectoryResult(relevantNodes, clusterTopology, dataDirectoryResult);
}
using (_serverStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, _responseBodyStream))
{
context.Write(writer, dataDirectoryResult.ToJson());
}
}
private static void UpdateDatabaseInfo(BlittableJsonReaderObject databaseRecord, ServerStore serverStore, string databaseName, DrivesUsage existingDrivesUsage,
DatabaseInfoItem databaseInfoItem)
{
DatabaseInfo databaseInfo = null;
if (serverStore.DatabaseInfoCache.TryGet(databaseName,
databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)) == false)
{
return;
}
Debug.Assert(databaseInfo != null);
var databaseTopology = serverStore.Cluster.ReadDatabaseTopology(databaseRecord);
databaseRecord.TryGet(nameof(DatabaseRecord.Indexes), out BlittableJsonReaderObject indexes);
var indexesCount = indexes?.Count ?? 0;
databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0;
databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? indexesCount;
databaseInfoItem.ReplicationFactor = databaseTopology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
if (databaseInfo.MountPointsUsage == null)
{
return;
}
foreach (var mountPointUsage in databaseInfo.MountPointsUsage)
{
var driveName = mountPointUsage.DiskSpaceResult.DriveName;
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(
mountPointUsage.DiskSpaceResult.DriveName,
new DriveInfoBase
{
DriveName = driveName
});
if (diskSpaceResult != null)
{
// update the latest drive info
mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult
{
DriveName = diskSpaceResult.DriveName,
VolumeLabel = diskSpaceResult.VolumeLabel,
TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
TotalSizeInBytes = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
};
}
UpdateMountPoint(serverStore.Configuration.Storage, mountPointUsage, databaseName, existingDrivesUsage);
}
}
public static void AllocateFileSpace(StorageEnvironmentOptions options, int fd, long size, string file)
{
bool usingLseek;
var result = Syscall.AllocateFileSpace(fd, size, file, out usingLseek);
if (result == (int)Errno.ENOSPC)
{
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(file);
throw new DiskFullException(file, size, diskSpaceResult?.TotalFreeSpace.GetValue(SizeUnit.Bytes));
}
if (result != 0)
{
Syscall.ThrowLastError(result, $"posix_fallocate(\"{file}\", {size})");
}
}
protected override Gauge32 GetData()
{
if (_store.Configuration.Core.RunInMemory)
return Empty;
var result = DiskSpaceChecker.GetDiskSpaceInfo(_store.Configuration.Core.DataDirectory.FullPath);
if (result == null)
return Empty;
var total = Convert.ToDecimal(result.TotalSize.GetValue(SizeUnit.Megabytes));
var totalFree = Convert.ToDecimal(result.TotalFreeSpace.GetValue(SizeUnit.Megabytes));
var percentage = Convert.ToInt32(Math.Round((totalFree / total) * 100, 0, MidpointRounding.ToEven));
return new Gauge32(percentage);
}
protected override Gauge32 GetData(DocumentDatabase database)
{
if (database.Configuration.Core.RunInMemory)
{
return(Empty);
}
var result = DiskSpaceChecker.GetDiskSpaceInfo(database.Configuration.Core.DataDirectory.FullPath);
if (result == null)
{
return(Empty);
}
return(new Gauge32(result.TotalFreeSpace.GetValue(SizeUnit.Megabytes)));
}
public static void SetFileLength(SafeFileHandle fileHandle, long length)
{
if (SetFilePointerEx(fileHandle, length, IntPtr.Zero, Win32NativeFileMoveMethod.Begin) == false)
{
var exception = new Win32Exception(Marshal.GetLastWin32Error());
var filePath = GetFilePath();
throw new IOException($"Could not move the pointer of file {filePath}", exception);
}
if (SetEndOfFile(fileHandle) == false)
{
var lastError = Marshal.GetLastWin32Error();
var filePath = GetFilePath();
if (lastError == (int)Win32NativeFileErrors.ERROR_DISK_FULL)
{
var driveInfo = DiskSpaceChecker.GetDiskSpaceInfo(filePath);
throw new DiskFullException(filePath, length, driveInfo?.TotalFreeSpace.GetValue(SizeUnit.Bytes));
}
var exception = new Win32Exception(lastError);
if (lastError == (int)Win32NativeFileErrors.ERROR_NOT_READY ||
lastError == (int)Win32NativeFileErrors.ERROR_FILE_NOT_FOUND)
{
throw new IOException($"Could not set the size of file {filePath} because it is inaccessible.", exception);
}
throw new IOException($"Could not set the size of file {filePath} to {Sizes.Humane(length)}", exception);
}
string GetFilePath()
{
var filePath = new StringBuilder(256);
while (GetFinalPathNameByHandle(fileHandle, filePath, filePath.Capacity, 0) > filePath.Capacity &&
filePath.Capacity < 32767) // max unicode path length
{
filePath.EnsureCapacity(filePath.Capacity * 2);
}
return(filePath.ToString());
}
}
public void AssertCanContinueWriting()
{
var diskInfoResult = DiskSpaceChecker.GetDiskSpaceInfo(_path);
if (diskInfoResult == null)
{
return;
}
var freeSpaceInBytes = diskInfoResult.TotalFreeSpace.GetValue(SizeUnit.Bytes);
if (freeSpaceInBytes > _availableSpaceWhenEventOccurred)
{
return;
}
_edi.Throw();
}
public static void SetFileLength(SafeFileHandle fileHandle, long length)
{
if (SetFilePointerEx(fileHandle, length, IntPtr.Zero, Win32NativeFileMoveMethod.Begin) == false)
{
var exception = new Win32Exception(Marshal.GetLastWin32Error());
var filePath = GetFilePath();
throw new IOException($"Could not move the pointer of file {filePath}", exception);
}
if (SetEndOfFile(fileHandle) == false)
{
var lastError = Marshal.GetLastWin32Error();
var filePath = GetFilePath();
if (lastError == (int)Win32NativeFileErrors.ERROR_DISK_FULL)
{
var driveInfo = DiskSpaceChecker.GetDiskSpaceInfo(filePath);
throw new DiskFullException(filePath, length, driveInfo?.TotalFreeSpace.GetValue(SizeUnit.Bytes), new Win32Exception(lastError).Message);
}
var exception = new Win32Exception(lastError);
if (lastError == (int)Win32NativeFileErrors.ERROR_NOT_READY ||
lastError == (int)Win32NativeFileErrors.ERROR_FILE_NOT_FOUND)
{
throw new IOException($"Could not set the size of file {filePath} because it is inaccessible.", exception);
}
throw new IOException($"Could not set the size of file {filePath} to {Sizes.Humane(length)}", exception);
}
string GetFilePath()
{
try
{
return(DiskSpaceChecker.GetWindowsRealPathByHandle(fileHandle.DangerousGetHandle()));
}
catch
{
return(null);
}
}
}
public static unsafe void AllocateFileSpace(StorageEnvironmentOptions options, int fd, long size, string file)
{
bool usingLseek;
var result = Syscall.AllocateFileSpace(fd, size, file, out usingLseek);
if (result == (int)Errno.ENOSPC)
{
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(file);
// Use Pal's detailed error string (until PosixHelper will be entirely removed)
var nativeMsg = PalHelper.GetNativeErrorString(result, "Failed to AllocateFileSpace (PosixHelper)", out _);
throw new DiskFullException(file, size, diskSpaceResult?.TotalFreeSpace.GetValue(SizeUnit.Bytes), nativeMsg);
}
if (result != 0)
{
Syscall.ThrowLastError(result, $"posix_fallocate(\"{file}\", {size})");
}
}
public static void AssertFreeSpaceForSnapshot(string directoryPath, long sizeInBytes, string action, Logger logger)
{
var destinationDriveInfo = DiskSpaceChecker.GetDiskSpaceInfo(directoryPath);
if (destinationDriveInfo == null)
{
if (logger.IsInfoEnabled)
{
logger.Info($"Couldn't find the disk space info for path: {directoryPath}");
}
return;
}
var desiredFreeSpace = Size.Min(new Size(512, SizeUnit.Megabytes), destinationDriveInfo.TotalSize * 0.01) + new Size(sizeInBytes, SizeUnit.Bytes);
if (destinationDriveInfo.TotalFreeSpace < desiredFreeSpace)
{
throw new InvalidOperationException($"Not enough free space to {action}. " +
$"Required space {desiredFreeSpace}, available space: {destinationDriveInfo.TotalFreeSpace}");
}
}
public RvnMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool canPrefetchAhead = true, bool usePageProtection = false, bool deleteOnClose = false)
: base(options, canPrefetchAhead, usePageProtection)
{
DeleteOnClose = deleteOnClose;
FileName = file;
var copyOnWriteMode = options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename);
_logger = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}");
if (initialFileSize.HasValue == false || initialFileSize.Value == 0)
{
initialFileSize = Math.Max(SysInfo.PageSize * 16, 64 * 1024);
}
if (initialFileSize % SysInfo.PageSize != 0)
{
initialFileSize += SysInfo.PageSize - initialFileSize % SysInfo.PageSize;
}
Debug.Assert(file != null);
var mmapOptions = copyOnWriteMode ? MmapOptions.CopyOnWrite : MmapOptions.None;
if (DeleteOnClose)
{
mmapOptions |= MmapOptions.DeleteOnClose;
}
var rc = rvn_create_and_mmap64_file(
file.FullPath,
initialFileSize.Value,
mmapOptions,
out _handle,
out var baseAddress,
out _totalAllocationSize,
out var errorCode);
if (rc != FailCodes.Success)
{
try
{
PalHelper.ThrowLastError(rc, errorCode, $"rvn_create_and_mmap64_file failed on {rc} for '{file.FullPath}'");
}
catch (DiskFullException dfEx)
{
var diskSpaceResult = DiskSpaceChecker.GetDiskSpaceInfo(file.FullPath);
throw new DiskFullException(file.FullPath, initialFileSize.Value, diskSpaceResult?.TotalFreeSpace.GetValue(SizeUnit.Bytes), dfEx.Message);
}
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
NativeMemory.RegisterFileMapping(FileName.FullPath, new IntPtr(baseAddress), _totalAllocationSize, GetAllocatedInBytes);
var allocationInfo = new PagerState.AllocationInfo
{
BaseAddress = (byte *)baseAddress,
Size = _totalAllocationSize,
MappedFile = null
};
var pager = new PagerState(this, Options.PrefetchSegmentSize, Options.PrefetchResetThreshold, allocationInfo);
SetPagerState(pager);
}
private DiskSpaceResult CalculateDiskSpaceInfo()
{
return(DiskSpaceChecker.GetDiskSpaceInfo(_server.ServerStore.Configuration.Core.DataDirectory.FullPath));
}
private (HashSet <LowDiskSpace> Disks, HashSet <StorageEnvironment> Environments) GetLowSpaceDisksAndRelevantEnvironments()
{
var lowSpaceDisks = new HashSet <LowDiskSpace>();
var environmentsRunningOnLowSpaceDisks = new HashSet <StorageEnvironment>();
var current = _databases.First;
while (current != null)
{
var database = current.Value;
var storageConfig = database.Configuration.Storage;
try
{
foreach (var item in database.GetAllStoragesEnvironment())
{
var driveInfo = item.Environment.Options.DriveInfoByPath?.Value;
var options = (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)item.Environment.Options;
var dataDisk = DiskSpaceChecker.GetDiskSpaceInfo(options.BasePath.FullPath, driveInfo?.BasePath);
AddEnvironmentIfLowSpace(dataDisk);
if (options.JournalPath != null)
{
var journalDisk = DiskSpaceChecker.GetDiskSpaceInfo(options.JournalPath.FullPath, driveInfo?.JournalPath);
if (dataDisk?.DriveName != journalDisk?.DriveName)
{
AddEnvironmentIfLowSpace(journalDisk);
}
}
if (options.TempPath != null)
{
var tempDisk = DiskSpaceChecker.GetDiskSpaceInfo(options.TempPath.FullPath, driveInfo?.TempPath);
if (dataDisk?.DriveName != tempDisk?.DriveName)
{
AddEnvironmentIfLowSpace(tempDisk);
}
}
void AddEnvironmentIfLowSpace(DiskSpaceResult diskSpace)
{
if (diskSpace == null)
{
return;
}
if (IsLowSpace(diskSpace.TotalFreeSpace, diskSpace.TotalSize, storageConfig, out var reason, SimulateLowDiskSpace) == false)
{
return;
}
var lowSpace = new LowDiskSpace(diskSpace, reason);
lowSpaceDisks.Add(lowSpace);
environmentsRunningOnLowSpaceDisks.Add(item.Environment);
}
}
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"Failed to check disk usage for '{database.Name}' database", e);
}
}
current = current.Next;
}
return(lowSpaceDisks, environmentsRunningOnLowSpaceDisks);
}
