public Posix32BitsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool usePageProtection = false) : base(options, usePageProtection) { _options = options; FileName = file; if (Options.CopyOnWriteMode) { ThrowNotSupportedOption(file.FullPath); } _copyOnWriteMode = options.CopyOnWriteMode && file.FullPath.EndsWith(Constants.DatabaseFilename); _isSyncDirAllowed = Syscall.CheckSyncDirectoryAllowed(FileName.FullPath); PosixHelper.EnsurePathExists(FileName.FullPath); Debug.Assert(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) == false); // O_LARGEFILE not exists in mac and supported by default (however we do not run on mac 32bit..) _fd = Syscall.open(file.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT | PerPlatformValues.OpenFlags.O_LARGEFILE, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + file); } _totalAllocationSize = GetFileSize(); if (_totalAllocationSize == 0 && initialFileSize.HasValue) { _totalAllocationSize = NearestSizeToAllocationGranularity(initialFileSize.Value); } if (_totalAllocationSize == 0 || _totalAllocationSize % AllocationGranularity != 0 || _totalAllocationSize != GetFileSize()) { _totalAllocationSize = NearestSizeToAllocationGranularity(_totalAllocationSize); PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, file.FullPath); } if (_isSyncDirAllowed && Syscall.SyncDirectory(file.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "sync dir for " + file); } NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; SetPagerState(new PagerState(this) { Files = null, MapBase = null, AllocationInfos = new PagerState.AllocationInfo[0] }); }
public override void ReleaseAllocationInfo(byte *baseAddress, long size) { var ptr = new IntPtr(baseAddress); var result = Syscall.munmap(ptr, (UIntPtr)size); if (result == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } NativeMemory.UnregisterFileMapping(FileName, ptr, size); }
public unsafe void WriteBuffer(long position, byte *srcPosition, int sizeToWrite) { var offset = Convert.ToUInt64(sizeToWrite); var result = Syscall.pwrite(_fd, srcPosition, (ulong)sizeToWrite, (long)offset); if (result == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } }
public override void Sync() { //TODO: Is it worth it to change to just one call for msync for the entire file? foreach (var alloc in PagerState.AllocationInfos) { var result = Syscall.msync(new IntPtr(alloc.BaseAddress), (ulong)alloc.Size, MsyncFlags.MS_SYNC); if (result == -1) { PosixHelper.ThrowLastError(Marshal.GetLastWin32Error()); } } }
private long GetFileSize() { Stat buf; var result = Syscall.fstat(_fd, out buf); if (result == -1) { PosixHelper.ThrowLastError(Marshal.GetLastWin32Error()); } var fileSize = buf.st_size; return(fileSize); }
public PosixMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool usePageProtection = false) : base(options, canPrefetchAhead: true, usePageProtection: usePageProtection) { _options = options; FileName = file; _copyOnWriteMode = options.CopyOnWriteMode && file.FullPath.EndsWith(Constants.DatabaseFilename); _isSyncDirAllowed = Syscall.CheckSyncDirectoryAllowed(FileName.FullPath); PosixHelper.EnsurePathExists(FileName.FullPath); _fd = Syscall.open(file.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + file); } SysPageSize = Syscall.sysconf(PerPlatformValues.SysconfNames._SC_PAGESIZE); if (SysPageSize <= 0) // i.e. -1 because _SC_PAGESIZE defined differently on various platforms { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "Got SysPageSize <= 0 for " + FileName); } _totalAllocationSize = GetFileSize(); if (_totalAllocationSize == 0 && initialFileSize.HasValue) { _totalAllocationSize = NearestSizeToPageSize(initialFileSize.Value); } if (_totalAllocationSize == 0 || _totalAllocationSize % SysPageSize != 0 || _totalAllocationSize != GetFileSize()) { _totalAllocationSize = NearestSizeToPageSize(_totalAllocationSize); PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, file.FullPath); } if (_isSyncDirAllowed && Syscall.SyncDirectory(file.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "sync dir for " + file); } NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; SetPagerState(CreatePagerState()); }
public unsafe void WriteGather(long position, IntPtr[] pages) { if (pages.Length == 0) { return; // nothing to do } var start = 0; const int IOV_MAX = 1024; while (start < pages.Length) { var byteLen = 0L; var locs = new List <Iovec> { new Iovec { iov_base = pages[start], iov_len = (ulong)_options.PageSize } }; start++; byteLen += _options.PageSize; for (int i = 1; i < pages.Length && locs.Count < IOV_MAX; i++, start++) { byteLen += _options.PageSize; var cur = locs[locs.Count - 1]; if (((byte *)cur.iov_base.ToPointer() + cur.iov_len) == (byte *)pages[i].ToPointer()) { cur.iov_len = cur.iov_len + (ulong)_options.PageSize; locs[locs.Count - 1] = cur; } else { locs.Add(new Iovec { iov_base = pages[i], iov_len = (ulong)_options.PageSize }); } } var result = Syscall.pwritev(_fd, locs.ToArray(), position); position += byteLen; if (result == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } } }
public PosixJournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long journalSize) { _options = options; _filename = filename; _maxNumberOf4KbPerSingleWrite = int.MaxValue / (4 * Constants.Size.Kilobyte); _fd = Syscall.open(filename.FullPath, OpenFlags.O_WRONLY | options.PosixOpenFlags | PerPlatformValues.OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + filename); } if (RunningOnMacOsx) { // mac doesn't support O_DIRECT, we fcntl instead: var rc = Syscall.fcntl(_fd, FcntlCommands.F_NOCACHE, (IntPtr)1); if (rc != 0) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when fcntl F_NOCACHE for " + filename); } } var length = new FileInfo(filename.FullPath).Length; if (length < journalSize) { length = journalSize; try { PosixHelper.AllocateFileSpace(options, _fd, journalSize, filename.FullPath); } catch (Exception) { Syscall.close(_fd); throw; } } if (Syscall.CheckSyncDirectoryAllowed(_filename.FullPath) && Syscall.SyncDirectory(filename.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when syncing dir for on " + filename); } NumberOfAllocated4Kb = (int)(length / (4 * Constants.Size.Kilobyte)); }
public unsafe void WriteGather(long position, byte *[] pages) { var locs = new Iovec[pages.Length]; for (int i = 0; i < locs.Length; i++) { locs[i].iov_base = new IntPtr(pages[i]); locs[i].iov_len = AbstractPager.PageSize; } var result = Syscall.pwritev(_fd, locs, position); if (result == -1) { PosixHelper.ThrowLastError(Marshal.GetLastWin32Error()); } }
protected override PagerState AllocateMorePages(long newLength) { if (Disposed) { ThrowAlreadyDisposedException(); } var newLengthAfterAdjustment = NearestSizeToPageSize(newLength); if (newLengthAfterAdjustment <= _totalAllocationSize) { return(null); } var allocationSize = newLengthAfterAdjustment - _totalAllocationSize; PosixHelper.AllocateFileSpace(_fd, (ulong)(_totalAllocationSize + allocationSize)); if (_isSyncDirAllowed && PosixHelper.SyncDirectory(_file) == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } PagerState newPagerState = CreatePagerState(); if (newPagerState == null) { var errorMessage = string.Format( "Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes", (_totalAllocationSize + allocationSize)); throw new OutOfMemoryException(errorMessage); } newPagerState.DebugVerify(newLengthAfterAdjustment); var tmp = PagerState; PagerState = newPagerState; tmp.Release(); //replacing the pager state --> so one less reference for it _totalAllocationSize += allocationSize; NumberOfAllocatedPages = _totalAllocationSize / _pageSize; return(newPagerState); }
protected internal override PagerState AllocateMorePages(long newLength) { if (DisposeOnceRunner.Disposed) { ThrowAlreadyDisposedException(); } var newLengthAfterAdjustment = NearestSizeToPageSize(newLength); if (newLengthAfterAdjustment <= _totalAllocationSize) { return(null); } var allocationSize = newLengthAfterAdjustment - _totalAllocationSize; PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize + allocationSize, FileName.FullPath); if (DeleteOnClose == false && _isSyncDirAllowed && Syscall.SyncDirectory(FileName.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err); } PagerState newPagerState = CreatePagerState(); if (newPagerState == null) { var errorMessage = string.Format( "Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes", (_totalAllocationSize + allocationSize)); throw new OutOfMemoryException(errorMessage); } newPagerState.DebugVerify(newLengthAfterAdjustment); newPagerState.CopyPrefetchState(this._pagerState); SetPagerState(newPagerState); _totalAllocationSize += allocationSize; NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; return(newPagerState); }
public override void AllocateMorePages(Transaction tx, long newLength) { ThrowObjectDisposedIfNeeded(); var newLengthAfterAdjustment = NearestSizeToPageSize(newLength); if (newLengthAfterAdjustment <= _totalAllocationSize) //nothing to do { return; } var allocationSize = newLengthAfterAdjustment - _totalAllocationSize; PosixHelper.AllocateFileSpace(_fd, (ulong)(_totalAllocationSize + allocationSize)); _totalAllocationSize += allocationSize; PagerState newPagerState = CreatePagerState(); if (newPagerState == null) { var errorMessage = string.Format( "Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes", (_totalAllocationSize + allocationSize)); throw new OutOfMemoryException(errorMessage); } newPagerState.DebugVerify(newLengthAfterAdjustment); if (tx != null) { newPagerState.AddRef(); tx.AddPagerState(newPagerState); } var tmp = PagerState; PagerState = newPagerState; tmp.Release(); //replacing the pager state --> so one less reference for it NumberOfAllocatedPages = _totalAllocationSize / PageSize; }
public PosixJournalWriter(string filename, long journalSize) { _filename = filename; _fd = Syscall.open(filename, OpenFlags.O_WRONLY | OpenFlags.O_SYNC | OpenFlags.O_CREAT, FilePermissions.ALLPERMS); if (_fd == -1) { PosixHelper.ThrowLastError(Marshal.GetLastWin32Error()); } var result = Syscall.posix_fallocate(_fd, 0, (ulong)journalSize); if (result != 0) { PosixHelper.ThrowLastError(result); } NumberOfAllocatedPages = journalSize / AbstractPager.PageSize; }
public PosixTempMemoryMapPager(string file, long?initialFileSize = null) { _file = file; _fd = Syscall.open(_file, OpenFlags.O_RDWR | OpenFlags.O_CREAT | OpenFlags.O_EXCL, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { PosixHelper.ThrowLastError(Marshal.GetLastWin32Error()); } DeleteOnClose = true; SysPageSize = Syscall.sysconf(SysconfName._SC_PAGESIZE); _totalAllocationSize = NearestSizeToPageSize(initialFileSize ?? _totalAllocationSize); PosixHelper.AllocateFileSpace(_fd, (ulong)_totalAllocationSize); NumberOfAllocatedPages = _totalAllocationSize / PageSize; PagerState.Release(); PagerState = CreatePagerState(); }
public PosixMemoryMapPager(StorageEnvironmentOptions options, string file, long?initialFileSize = null, bool usePageProtection = false) : base(options, usePageProtection) { FileName = file; _copyOnWriteMode = options.CopyOnWriteMode && file.EndsWith(Constants.DatabaseFilename); _isSyncDirAllowed = PosixHelper.CheckSyncDirectoryAllowed(FileName); PosixHelper.EnsurePathExists(FileName); _fd = Syscall.open(file, OpenFlags.O_RDWR | OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when opening " + file); } SysPageSize = Syscall.sysconf(SysconfName._SC_PAGESIZE); _totalAllocationSize = GetFileSize(); if (_totalAllocationSize == 0 && initialFileSize.HasValue) { _totalAllocationSize = NearestSizeToPageSize(initialFileSize.Value); } if (_totalAllocationSize == 0 || _totalAllocationSize % SysPageSize != 0 || _totalAllocationSize != GetFileSize()) { _totalAllocationSize = NearestSizeToPageSize(_totalAllocationSize); PosixHelper.AllocateFileSpace(_fd, (ulong)_totalAllocationSize, file); } if (_isSyncDirAllowed && PosixHelper.SyncDirectory(file) == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "sync dir for " + file); } NumberOfAllocatedPages = _totalAllocationSize / PageSize; SetPagerState(CreatePagerState()); }
public PosixMemoryMapPager(int pageSize, string file, long?initialFileSize = null) : base(pageSize) { _file = file; _isSyncDirAllowed = PosixHelper.CheckSyncDirectoryAllowed(_file); PosixHelper.EnsurePathExists(_file); _fd = Syscall.open(file, OpenFlags.O_RDWR | OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } SysPageSize = Syscall.sysconf(SysconfName._SC_PAGESIZE); _totalAllocationSize = GetFileSize(); if (_totalAllocationSize == 0 && initialFileSize.HasValue) { _totalAllocationSize = NearestSizeToPageSize(initialFileSize.Value); } if (_totalAllocationSize == 0 || _totalAllocationSize % SysPageSize != 0 || _totalAllocationSize != GetFileSize()) { _totalAllocationSize = NearestSizeToPageSize(_totalAllocationSize); PosixHelper.AllocateFileSpace(_fd, (ulong)_totalAllocationSize); } if (_isSyncDirAllowed && PosixHelper.SyncDirectory(file) == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } NumberOfAllocatedPages = _totalAllocationSize / _pageSize; PagerState.Release(); PagerState = CreatePagerState(); }
protected internal override PagerState AllocateMorePages(long newLength) { if (DisposeOnceRunner.Disposed) { ThrowAlreadyDisposedException(); } var newLengthAfterAdjustment = NearestSizeToPageSize(newLength); if (newLengthAfterAdjustment <= _totalAllocationSize) //nothing to do { return(null); } var allocationSize = newLengthAfterAdjustment - _totalAllocationSize; PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize + allocationSize, FileName.FullPath); _totalAllocationSize += allocationSize; PagerState newPagerState = CreatePagerState(); if (newPagerState == null) { var errorMessage = string.Format( "Unable to allocate more pages - unsuccessfully tried to allocate continuous block of virtual memory with size = {0:##,###;;0} bytes", (_totalAllocationSize + allocationSize)); throw new OutOfMemoryException(errorMessage); } newPagerState.DebugVerify(newLengthAfterAdjustment); SetPagerState(newPagerState); NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; return(newPagerState); }
private unsafe void WriteFile(long position, byte *p, int numberOfPages) { if (numberOfPages == 0) { return; // nothing to do } var nNumberOfBytesToWrite = (ulong)numberOfPages * (ulong)_options.PageSize; long actuallyWritten = 0; long result; using (_options.IoMetrics.MeterIoRate(_filename, IoMetrics.MeterType.JournalWrite, (long)nNumberOfBytesToWrite)) { do { result = Syscall.pwrite(_fd, p, nNumberOfBytesToWrite - (ulong)actuallyWritten, position); if (result < 1) { break; } actuallyWritten += result; p += actuallyWritten; } while ((ulong)actuallyWritten < nNumberOfBytesToWrite); } if (result == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when writing to " + _filename); } else if (result == 0) { var err = Marshal.GetLastWin32Error(); throw new IOException($"pwrite reported zero bytes written, after write of {actuallyWritten} bytes out of {nNumberOfBytesToWrite}. lastErrNo={err} on {_filename}"); } else if ((ulong)actuallyWritten != nNumberOfBytesToWrite) { var err = Marshal.GetLastWin32Error(); throw new IOException($"pwrite couln't write {nNumberOfBytesToWrite} to file. only {actuallyWritten} written. lastErrNo={err} on {_filename}"); } }
public void Truncate(long size) { var result = Syscall.ftruncate(_fd, (IntPtr)size); if (result == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when truncating " + _filename); } result = Syscall.fsync(_fd); if (result == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when fsycning " + _filename); } if (PosixHelper.CheckSyncDirectoryAllowed(_filename) && PosixHelper.SyncDirectory(_filename) == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when syncing dir for " + _filename); } }
public PosixJournalWriter(StorageEnvironmentOptions options, string filename, long journalSize) { _options = options; _filename = filename; _maxNumberOfPagesPerSingleWrite = int.MaxValue / _options.PageSize; _fd = Syscall.open(filename, OpenFlags.O_WRONLY | options.PosixOpenFlags | OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when opening " + filename); } int result; if ((options.SafePosixOpenFlags & PerPlatformValues.OpenFlags.O_DIRECT) == 0) { // fallocate doesn't supported, we'll use lseek instead result = Syscall.AllocateUsingLseek(_fd, journalSize); } else { result = Syscall.posix_fallocate(_fd, IntPtr.Zero, (UIntPtr)journalSize); } if (result != 0) { PosixHelper.ThrowLastError(result, "when allocating " + filename); } if (PosixHelper.CheckSyncDirectoryAllowed(_filename) && PosixHelper.SyncDirectory(filename) == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err, "when syncing dir for on " + filename); } NumberOfAllocatedPages = (int)(journalSize / _options.PageSize); }
protected internal override PagerState AllocateMorePages(long newLength) { var newLengthAfterAdjustment = NearestSizeToAllocationGranularity(newLength); if (newLengthAfterAdjustment <= _totalAllocationSize) { return(null); } var allocationSize = newLengthAfterAdjustment - _totalAllocationSize; PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize + allocationSize, FileName.FullPath); if (_isSyncDirAllowed && Syscall.SyncDirectory(FileName.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err); } _totalAllocationSize += allocationSize; NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; return(null); }
public PosixTempMemoryMapPager(StorageEnvironmentOptions options, string file, long?initialFileSize = null) : base(options) { FileName = file; PosixHelper.EnsurePathExists(file); _fd = Syscall.open(FileName, OpenFlags.O_RDWR | OpenFlags.O_CREAT | OpenFlags.O_EXCL, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); PosixHelper.ThrowLastError(err); } DeleteOnClose = true; SysPageSize = Syscall.sysconf(SysconfName._SC_PAGESIZE); _totalAllocationSize = NearestSizeToPageSize(initialFileSize ?? _totalAllocationSize); PosixHelper.AllocateFileSpace(_fd, (ulong)_totalAllocationSize, FileName); NumberOfAllocatedPages = _totalAllocationSize / PageSize; SetPagerState(CreatePagerState()); }