public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path) { using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None)) { if (fs.Length != sizeof(FileHeader)) { return(false); // wrong file size } var ptr = (byte *)header; int remaining = sizeof(FileHeader); while (remaining > 0) { int read; if (Win32NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false) { throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to read file " + path); } if (read == 0) { return(false); // we should be reading _something_ here, if we can't, then it is an error and we assume corruption } ptr += read; remaining -= read; } return(true); } }
public PosixTempMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null) : base(options, canPrefetchAhead: false) { _options = options; FileName = file; PosixHelper.EnsurePathExists(file.FullPath); _fd = Syscall.open(FileName.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT | PerPlatformValues.OpenFlags.O_EXCL, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "Cannot open " + FileName); } DeleteOnClose = true; SysPageSize = Syscall.sysconf(PerPlatformValues.SysconfNames._SC_PAGESIZE); if (SysPageSize <= 0) // i.e. -1 because _SC_PAGESIZE defined differently on various platforms { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "Got SysPageSize <= 0 for " + FileName); } _totalAllocationSize = NearestSizeToPageSize(initialFileSize ?? _totalAllocationSize); PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, FileName.FullPath); NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; SetPagerState(CreatePagerState()); }
private void Restore(StorageEnvironment env, string singleBackupFile) { using (env.Journal.Applicator.TakeFlushingLock()) { env.FlushLogToDataFile(); var transactionPersistentContext = new TransactionPersistentContext(true); using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite)) { using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read, System.Text.Encoding.UTF8)) { if (package.Entries.Count == 0) { return; } var toDispose = new List <IDisposable>(); var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName; var tempDirSettings = new VoronPathSetting(tempDir); Restore(env, package.Entries, tempDirSettings, toDispose, txw); } } } }
private List <TempBufferReport> GenerateTempBuffersReport(VoronPathSetting tempPath, VoronPathSetting journalPath) { var tempFiles = Directory.GetFiles(tempPath.FullPath, "*.buffers").Select(filePath => { var file = new FileInfo(filePath); return(new TempBufferReport { Name = file.Name, AllocatedSpaceInBytes = file.Length, Type = TempBufferType.Scratch }); }).ToList(); if (journalPath != null) { var recyclableJournals = Directory.GetFiles(journalPath.FullPath, $"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}.*").Select(filePath => { var file = new FileInfo(filePath); return(new TempBufferReport { Name = file.Name, AllocatedSpaceInBytes = file.Length, Type = TempBufferType.RecyclableJournal }); }).ToList(); tempFiles.AddRange(recyclableJournals); } return(tempFiles); }
public void ToFile(StorageEnvironment env, VoronPathSetting backupPath, CompressionLevel compression = CompressionLevel.Optimal, Action <string> infoNotify = null, Action backupStarted = null) { infoNotify = infoNotify ?? (s => { }); infoNotify("Voron backup db started"); using (var file = SafeFileStream.Create(backupPath.FullPath, FileMode.Create)) { using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true)) { infoNotify("Voron backup started"); var dataPager = env.Options.DataPager; var copier = new DataCopier(Constants.Storage.PageSize * 16); Backup(env, compression, infoNotify, backupStarted, dataPager, package, string.Empty, copier); file.Flush(true); // make sure that we fully flushed to disk } } infoNotify("Voron backup db finished"); }
public static StorageEnvironmentOptions ForPath(string path, string tempPath, string journalPath, IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification) { var pathSetting = new VoronPathSetting(path); var tempPathSetting = new VoronPathSetting(tempPath ?? GetTempPath(path)); var journalPathSetting = journalPath != null ? new VoronPathSetting(journalPath) : null; return(new DirectoryStorageEnvironmentOptions(pathSetting, tempPathSetting, journalPathSetting, ioChangesNotifications, catastrophicFailureNotification)); }
public Win32FileJournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long journalSize, Win32NativeFileAccess access = Win32NativeFileAccess.GenericWrite, Win32NativeFileShare shareMode = Win32NativeFileShare.Read) { try { _options = options; _filename = filename; _handle = Win32NativeFileMethods.CreateFile(filename.FullPath, access, shareMode, IntPtr.Zero, Win32NativeFileCreationDisposition.OpenAlways, options.WinOpenFlags, IntPtr.Zero); if (_handle.IsInvalid) { throw new IOException("When opening file " + filename, new Win32Exception(Marshal.GetLastWin32Error())); } var length = new FileInfo(filename.FullPath).Length; if (length < journalSize) { try { Win32NativeFileMethods.SetFileLength(_handle, journalSize); } catch (Exception) { try { _handle?.Dispose(); _handle = null; File.Delete(_filename.FullPath); } catch (Exception) { // there's nothing we can do about it } throw; } length = journalSize; } NumberOfAllocated4Kb = (int)(length / (4 * Constants.Size.Kilobyte)); _nativeOverlapped = (NativeOverlapped *)NativeMemory.AllocateMemory(sizeof(NativeOverlapped)); _nativeOverlapped->InternalLow = IntPtr.Zero; _nativeOverlapped->InternalHigh = IntPtr.Zero; } catch { Dispose(); throw; } }
public void Restore(VoronPathSetting backupPath, VoronPathSetting voronDataDir, VoronPathSetting journalDir = null, Action <string> onProgress = null, CancellationToken cancellationToken = default) { using (var zip = ZipFile.Open(backupPath.FullPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8)) Restore(zip.Entries, voronDataDir, journalDir, onProgress, cancellationToken); }
public PosixJournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long journalSize) { try { _options = options; _filename = filename; _maxNumberOf4KbPerSingleWrite = int.MaxValue / (4 * Constants.Size.Kilobyte); _fd = Syscall.open(filename.FullPath, OpenFlags.O_WRONLY | options.PosixOpenFlags | PerPlatformValues.OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + filename); } if (RunningOnMacOsx) { // mac doesn't support O_DIRECT, we fcntl instead: var rc = Syscall.fcntl(_fd, FcntlCommands.F_NOCACHE, (IntPtr)1); if (rc != 0) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when fcntl F_NOCACHE for " + filename); } } var length = new FileInfo(filename.FullPath).Length; if (length < journalSize) { length = journalSize; try { PosixHelper.AllocateFileSpace(options, _fd, journalSize, filename.FullPath); } catch (Exception) { Syscall.close(_fd); throw; } } if (Syscall.CheckSyncDirectoryAllowed(_filename.FullPath) && Syscall.SyncDirectory(filename.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when syncing dir for on " + filename); } NumberOfAllocated4Kb = (int)(length / (4 * Constants.Size.Kilobyte)); } catch { Dispose(); throw; } }
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path) { bool syncIsNeeded = false; var fd = Syscall.open(path.FullPath, OpenFlags.O_WRONLY | PerPlatformValues.OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); try { if (fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + path); } int remaining = sizeof(FileHeader); FileInfo fi = new FileInfo(path.FullPath); if (fi.Length != remaining) { syncIsNeeded = true; } var ptr = ((byte *)header); while (remaining > 0) { var written = Syscall.write(fd, ptr, (ulong)remaining); if (written == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "writing to " + path); } remaining -= (int)written; ptr += written; } if (Syscall.FSync(fd) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "FSync " + path); } if (syncIsNeeded) { Syscall.FsyncDirectoryFor(path.FullPath); } } finally { if (fd != -1) { Syscall.close(fd); fd = -1; } } }
public Posix32BitsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool usePageProtection = false) : base(options, usePageProtection) { _options = options; FileName = file; if (Options.CopyOnWriteMode) { ThrowNotSupportedOption(file.FullPath); } _copyOnWriteMode = options.CopyOnWriteMode && file.FullPath.EndsWith(Constants.DatabaseFilename); _isSyncDirAllowed = Syscall.CheckSyncDirectoryAllowed(FileName.FullPath); PosixHelper.EnsurePathExists(FileName.FullPath); Debug.Assert(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) == false); // O_LARGEFILE not exists in mac and supported by default (however we do not run on mac 32bit..) _fd = Syscall.open(file.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT | PerPlatformValues.OpenFlags.O_LARGEFILE, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + file); } _totalAllocationSize = GetFileSize(); if (_totalAllocationSize == 0 && initialFileSize.HasValue) { _totalAllocationSize = NearestSizeToAllocationGranularity(initialFileSize.Value); } if (_totalAllocationSize == 0 || _totalAllocationSize % AllocationGranularity != 0 || _totalAllocationSize != GetFileSize()) { _totalAllocationSize = NearestSizeToAllocationGranularity(_totalAllocationSize); PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, file.FullPath); } if (_isSyncDirAllowed && Syscall.SyncDirectory(file.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "sync dir for " + file); } NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; SetPagerState(new PagerState(this) { Files = null, MapBase = null, AllocationInfos = new PagerState.AllocationInfo[0] }); }
private List <TempBufferReport> GenerateTempBuffersReport(VoronPathSetting tempPath) { return(Directory.GetFiles(tempPath.FullPath, "*.buffers").Select(filePath => { var file = new FileInfo(filePath); return new TempBufferReport { Name = file.Name, AllocatedSpaceInBytes = file.Length }; }).ToList()); }
public void FullBackupMustNotDeadlockWithFlush() { RequireFileBasedPager(); var random = new Random(2); var buffer = new byte[8192]; random.NextBytes(buffer); using (var tx = Env.WriteTransaction()) { var tree = tx.CreateTree("foo"); for (int i = 0; i < 5000; i++) { tree.Add("items/" + i, new MemoryStream(buffer)); } tx.Commit(); } var voronDataDir = new VoronPathSetting(DataDir); var backupCompleted = false; Exception backupException = null; Thread backup = new Thread(() => { try { BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup")); backupCompleted = true; } catch (Exception e) { backupException = e; } }); Env.Journal.Applicator.ForTestingPurposesOnly().OnApplyLogsToDataFileUnderFlushingLock += () => { backup.Start(); }; Env.FlushLogToDataFile(); backup.Join(TimeSpan.FromSeconds(60)); Assert.Null(backupException); Assert.True(backupCompleted); }
public PosixMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, bool usePageProtection = false) : base(options, canPrefetchAhead: true, usePageProtection: usePageProtection) { _options = options; FileName = file; _copyOnWriteMode = options.CopyOnWriteMode && file.FullPath.EndsWith(Constants.DatabaseFilename); _isSyncDirAllowed = Syscall.CheckSyncDirectoryAllowed(FileName.FullPath); PosixHelper.EnsurePathExists(FileName.FullPath); _fd = Syscall.open(file.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT, FilePermissions.S_IWUSR | FilePermissions.S_IRUSR); if (_fd == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "when opening " + file); } SysPageSize = Syscall.sysconf(PerPlatformValues.SysconfNames._SC_PAGESIZE); if (SysPageSize <= 0) // i.e. -1 because _SC_PAGESIZE defined differently on various platforms { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "Got SysPageSize <= 0 for " + FileName); } _totalAllocationSize = GetFileSize(); if (_totalAllocationSize == 0 && initialFileSize.HasValue) { _totalAllocationSize = NearestSizeToPageSize(initialFileSize.Value); } if (_totalAllocationSize == 0 || _totalAllocationSize % SysPageSize != 0 || _totalAllocationSize != GetFileSize()) { _totalAllocationSize = NearestSizeToPageSize(_totalAllocationSize); PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, file.FullPath); } if (_isSyncDirAllowed && Syscall.SyncDirectory(file.FullPath) == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err, "sync dir for " + file); } NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; SetPagerState(CreatePagerState()); }
public void Can_reuse_files_for_cache() { var path = new VoronPathSetting(NewDataPath()); var environment = new StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions(path, path, path, null, null); using (File.Create(TempFileCache.GetTempFileName(environment))) { } using (var cache = new TempFileCache(environment)) { Assert.Equal(1, cache.FilesCount); } }
public JournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long size, PalFlags.JournalMode mode = PalFlags.JournalMode.Safe) { _options = options; FileName = filename; _log = LoggingSource.Instance.GetLogger <JournalWriter>(options.BasePath.FullPath); var result = Pal.rvn_open_journal_for_writes(filename.FullPath, mode, size, options.SupportDurabilityFlags, out _writeHandle, out var actualSize, out var error); if (result != PalFlags.FailCodes.Success) { PalHelper.ThrowLastError(result, error, $"Attempted to open journal file - Path: {filename.FullPath} Size :{size}"); } NumberOfAllocated4Kb = (int)(actualSize / (4 * Constants.Size.Kilobyte)); }
private void EnsureMinimumSize(FileInfo fileInfo, VoronPathSetting path) { try { using (var stream = fileInfo.Open(FileMode.OpenOrCreate)) { stream.SetLength(InitialLogFileSize); } } catch (Exception e) { throw new InvalidOperationException( "Journal file " + path + " could not be opened because it's size is too small and we couldn't increase it", e); } }
protected StorageEnvironmentOptions(VoronPathSetting tempPath, IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification) { SafePosixOpenFlags = SafePosixOpenFlags | DefaultPosixFlags; DisposeWaitTime = TimeSpan.FromSeconds(15); TempPath = tempPath; ShouldUseKeyPrefix = name => false; MaxLogFileSize = ((sizeof(int) == IntPtr.Size ? 32 : 256) * Constants.Size.Megabyte); InitialLogFileSize = 64 * Constants.Size.Kilobyte; MaxScratchBufferSize = ((sizeof(int) == IntPtr.Size ? 32 : 256) * Constants.Size.Megabyte); MaxNumberOfPagesInJournalBeforeFlush = ((sizeof(int) == IntPtr.Size ? 4 : 32) * Constants.Size.Megabyte) / Constants.Storage.PageSize; IdleFlushTimeout = 5000; // 5 seconds OwnsPagers = true; IncrementalBackupEnabled = false; IoMetrics = new IoMetrics(256, 256, ioChangesNotifications); _log = LoggingSource.Instance.GetLogger <StorageEnvironment>(tempPath.FullPath); _catastrophicFailureNotification = catastrophicFailureNotification ?? new CatastrophicFailureNotification((e) => { if (_log.IsOperationsEnabled) { _log.Operations($"Catastrophic failure in {this}", e); } }); var shouldForceEnvVar = Environment.GetEnvironmentVariable("VORON_INTERNAL_ForceUsing32BitsPager"); bool result; if (bool.TryParse(shouldForceEnvVar, out result)) { ForceUsing32BitsPager = result; } }
private AbstractPager OpenPagerInternal(VoronPathSetting filename) { if (RunningOnPosix) { if (RunningOn32Bits) { return(new Posix32BitsMemoryMapPager(this, filename)); } return(new PosixMemoryMapPager(this, filename)); } if (RunningOn32Bits) { return(new Windows32BitsMemoryMapPager(this, filename)); } return(new WindowsMemoryMapPager(this, filename)); }
private void Restore(StorageEnvironment env, IEnumerable <ZipArchiveEntry> entries) { using (env.Journal.Applicator.TakeFlushingLock()) { env.FlushLogToDataFile(); var transactionPersistentContext = new TransactionPersistentContext(true); using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite)) { var toDispose = new List <IDisposable>(); var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName; var tempDirSettings = new VoronPathSetting(tempDir); Restore(env, entries, tempDirSettings, toDispose, txw); } } }
public void Restore(IEnumerable <ZipArchiveEntry> entries, VoronPathSetting voronDataDir, VoronPathSetting journalDir = null, Action <string> onProgress = null, CancellationToken cancellationToken = default) { journalDir = journalDir ?? voronDataDir.Combine("Journals"); if (Directory.Exists(voronDataDir.FullPath) == false) { Directory.CreateDirectory(voronDataDir.FullPath); } if (Directory.Exists(journalDir.FullPath) == false) { Directory.CreateDirectory(journalDir.FullPath); } onProgress?.Invoke("Starting snapshot restore"); foreach (var entry in entries) { var dst = string.Equals(Path.GetExtension(entry.Name), ".journal", StringComparison.OrdinalIgnoreCase) ? journalDir : voronDataDir; var sw = Stopwatch.StartNew(); if (Directory.Exists(dst.FullPath) == false) { Directory.CreateDirectory(dst.FullPath); } using (var input = entry.Open()) using (var output = SafeFileStream.Create(dst.Combine(entry.Name).FullPath, FileMode.CreateNew)) { input.CopyTo(output, cancellationToken); } onProgress?.Invoke($"Restored file: '{entry.Name}' to: '{dst}', " + $"size in bytes: {entry.Length:#,#;;0}, " + $"took: {sw.ElapsedMilliseconds:#,#;;0}ms"); } }
public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path) { var fd = Syscall.open(path.FullPath, OpenFlags.O_RDONLY, FilePermissions.S_IRUSR); try { if (fd == -1) { var lastError = Marshal.GetLastWin32Error(); if (((Errno)lastError) == Errno.EACCES) { return(false); } Syscall.ThrowLastError(lastError); } int remaining = sizeof(FileHeader); var ptr = ((byte *)header); while (remaining > 0) { var read = Syscall.read(fd, ptr, (ulong)remaining); if (read == -1) { var err = Marshal.GetLastWin32Error(); Syscall.ThrowLastError(err); } if (read == 0) { return(false); // truncated file? } remaining -= (int)read; ptr += read; } return(true); } finally { if (fd != -1) { Syscall.close(fd); fd = -1; } } }
public PureMemoryStorageEnvironmentOptions(string name, VoronPathSetting tempPath, IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification) : base(tempPath, ioChangesNotifications, catastrophicFailureNotification) { _name = name; _instanceId = Interlocked.Increment(ref _counter); var guid = Guid.NewGuid(); var filename = $"ravendb-{Process.GetCurrentProcess().Id}-{_instanceId}-data.pager-{guid}"; WinOpenFlags = Win32NativeFileAttributes.Temporary | Win32NativeFileAttributes.DeleteOnClose; if (Directory.Exists(tempPath.FullPath) == false) { Directory.CreateDirectory(tempPath.FullPath); } _dataPager = new Lazy <AbstractPager>(() => GetTempMemoryMapPager(this, TempPath.Combine(filename), InitialFileSize, Win32NativeFileAttributes.RandomAccess | Win32NativeFileAttributes.DeleteOnClose | Win32NativeFileAttributes.Temporary), true); }
public void Skip_files_that_are_in_use() { var path = new VoronPathSetting(NewDataPath()); var environment = new StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions(path, path, path, null, null); for (var i = 0; i < TempFileCache.MaxFilesToKeepInCache; i++) { using (File.Create(TempFileCache.GetTempFileName(environment))) { } } using (File.Create(Path.Combine(environment.TempPath.FullPath, TempFileCache.FilePrefix + "Z" + StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions.TempFileExtension))) { using (new TempFileCache(environment)) { } } }
private static bool TryMoveJournals(VoronPathSetting basePath, VoronPathSetting journalsPath) { var journalsInRoot = Directory.GetFiles(basePath.FullPath, "*.journal", SearchOption.TopDirectoryOnly); if (journalsInRoot.Length == 0) { return(false); } foreach (var journalFile in journalsInRoot) { var journalFileInfo = new FileInfo(journalFile); var source = basePath.Combine(journalFileInfo.Name); var destination = journalsPath.Combine(journalFileInfo.Name); File.Move(source.FullPath, destination.FullPath); } return(true); }
private static void BackupAndDeleteFile(VoronPathSetting path) { if (File.Exists(path.FullPath) == false) { return; } var count = 0; while (true) { var filePath = $"{path.FullPath}.{count++}.bak"; if (File.Exists(filePath)) { continue; } File.Move(path.FullPath, filePath); break; } }
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path) { using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None)) { var ptr = (byte *)header; int remaining = sizeof(FileHeader); while (remaining > 0) { int written; if (Win32NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out written, null) == false) { throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to write to file " + path); } ptr += written; remaining -= written; } if (Win32NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle) == false) { throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to Flush File Buffers (sync) of file " + path); } } }
public DirectoryStorageEnvironmentOptions(VoronPathSetting basePath, VoronPathSetting tempPath, VoronPathSetting journalPath, IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification) : base(tempPath ?? basePath, ioChangesNotifications, catastrophicFailureNotification) { _basePath = basePath; _journalPath = journalPath ?? basePath; if (Directory.Exists(_basePath.FullPath) == false) { Directory.CreateDirectory(_basePath.FullPath); } if (Equals(_basePath, tempPath) == false && Directory.Exists(TempPath.FullPath) == false) { Directory.CreateDirectory(TempPath.FullPath); } if (Equals(_journalPath, tempPath) == false && Directory.Exists(_journalPath.FullPath) == false) { Directory.CreateDirectory(_journalPath.FullPath); } _dataPager = new Lazy <AbstractPager>(() => { FilePath = _basePath.Combine(Constants.DatabaseFilename); return(GetMemoryMapPager(this, InitialFileSize, FilePath, usePageProtection: true)); }); // have to be before the journal check, so we'll fail on files in use // TODO: Need to verify behavior on Linux, might need to maintain a file lock DeleteAllTempBuffers(); GatherRecyclableJournalFiles(); // if there are any (e.g. after a rude db shut down) let us reuse them }
public WindowsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null, Win32NativeFileAttributes fileAttributes = Win32NativeFileAttributes.Normal, Win32NativeFileAccess access = Win32NativeFileAccess.GenericRead | Win32NativeFileAccess.GenericWrite, bool usePageProtection = false) : base(options, !fileAttributes.HasFlag(Win32NativeFileAttributes.Temporary), usePageProtection) { SYSTEM_INFO systemInfo; GetSystemInfo(out systemInfo); FileName = file; _logger = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}"); _access = access; _copyOnWriteMode = Options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename); if (_copyOnWriteMode) { _memoryMappedFileAccess = MemoryMappedFileAccess.Read | MemoryMappedFileAccess.CopyOnWrite; fileAttributes = Win32NativeFileAttributes.Readonly; _access = Win32NativeFileAccess.GenericRead; } else { _memoryMappedFileAccess = _access == Win32NativeFileAccess.GenericRead ? MemoryMappedFileAccess.Read : MemoryMappedFileAccess.ReadWrite; } _fileAttributes = fileAttributes; _handle = Win32NativeFileMethods.CreateFile(file.FullPath, access, Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete, IntPtr.Zero, Win32NativeFileCreationDisposition.OpenAlways, fileAttributes, IntPtr.Zero); if (_handle.IsInvalid) { int lastWin32ErrorCode = Marshal.GetLastWin32Error(); throw new IOException("Failed to open file storage of WinMemoryMapPager for " + file, new Win32Exception(lastWin32ErrorCode)); } _fileInfo = new FileInfo(file.FullPath); var drive = _fileInfo.Directory.Root.Name.TrimEnd('\\'); try { if (PhysicalDrivePerMountCache.TryGetValue(drive, out UniquePhysicalDriveId) == false) { UniquePhysicalDriveId = GetPhysicalDriveId(drive); } if (_logger.IsInfoEnabled) { _logger.Info($"Physical drive '{drive}' unique id = '{UniquePhysicalDriveId}' for file '{file}'"); } } catch (Exception ex) { UniquePhysicalDriveId = 0; if (_logger.IsInfoEnabled) { _logger.Info($"Failed to determine physical drive Id for drive letter '{drive}', file='{file}'", ex); } } var streamAccessType = _access == Win32NativeFileAccess.GenericRead ? FileAccess.Read : FileAccess.ReadWrite; _fileStream = SafeFileStream.Create(_handle, streamAccessType); _totalAllocationSize = _fileInfo.Length; if ((access & Win32NativeFileAccess.GenericWrite) == Win32NativeFileAccess.GenericWrite || (access & Win32NativeFileAccess.GenericAll) == Win32NativeFileAccess.GenericAll || (access & Win32NativeFileAccess.FILE_GENERIC_WRITE) == Win32NativeFileAccess.FILE_GENERIC_WRITE) { var fileLength = _fileStream.Length; if (fileLength == 0 && initialFileSize.HasValue) { fileLength = initialFileSize.Value; } if (_fileStream.Length == 0 || (fileLength % AllocationGranularity != 0)) { fileLength = NearestSizeToAllocationGranularity(fileLength); Win32NativeFileMethods.SetFileLength(_handle, fileLength); } _totalAllocationSize = fileLength; } NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize; SetPagerState(CreatePagerState()); }
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath, Action <IOperationProgress> onProgress, RestoreResult restoreResult) { Debug.Assert(onProgress != null); RestoreSettings restoreSettings = null; var fullBackupPath = GetBackupPath(backupPath); using (var zip = await GetZipArchiveForSnapshot(fullBackupPath)) { foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length))) { var directory = zipEntries.Key; if (string.IsNullOrWhiteSpace(directory)) { foreach (var zipEntry in zipEntries) { if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase)) { using (var entryStream = zipEntry.Open()) { var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey) : null; using (var stream = GetInputStream(entryStream, snapshotEncryptionKey)) { var json = context.Read(stream, "read database settings for restore"); json.BlittableValidation(); restoreSettings = JsonDeserializationServer.RestoreSettings(json); restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName; DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration); if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false) { throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!"); } if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey) { throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!"); } } } } } continue; } var voronDataDirectory = new VoronPathSetting(RestoreFromConfiguration.DataDirectory); var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase) ? voronDataDirectory : voronDataDirectory.Combine(directory); BackupMethods.Full.Restore( zipEntries, restoreDirectory, journalDir: null, onProgress: message => { restoreResult.AddInfo(message); restoreResult.SnapshotRestore.ReadCount++; onProgress.Invoke(restoreResult.Progress); }, cancellationToken: _operationCancelToken.Token); } } if (restoreSettings == null) { throw new InvalidDataException("Cannot restore the snapshot without the settings file!"); } return(restoreSettings); }