public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path)
{
using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
{
if (fs.Length != sizeof(FileHeader))
{
return(false); // wrong file size
}
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (Win32NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to read file " + path);
}
if (read == 0)
{
return(false); // we should be reading _something_ here, if we can't, then it is an error and we assume corruption
}
ptr += read;
remaining -= read;
}
return(true);
}
}
public PosixTempMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null)
: base(options, canPrefetchAhead: false)
{
_options = options;
FileName = file;
PosixHelper.EnsurePathExists(file.FullPath);
_fd = Syscall.open(FileName.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT | PerPlatformValues.OpenFlags.O_EXCL,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
if (_fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "Cannot open " + FileName);
}
DeleteOnClose = true;
SysPageSize = Syscall.sysconf(PerPlatformValues.SysconfNames._SC_PAGESIZE);
if (SysPageSize <= 0) // i.e. -1 because _SC_PAGESIZE defined differently on various platforms
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "Got SysPageSize <= 0 for " + FileName);
}
_totalAllocationSize = NearestSizeToPageSize(initialFileSize ?? _totalAllocationSize);
PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, FileName.FullPath);
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
SetPagerState(CreatePagerState());
}
private void Restore(StorageEnvironment env, string singleBackupFile)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
env.FlushLogToDataFile();
var transactionPersistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite))
{
using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
{
if (package.Entries.Count == 0)
{
return;
}
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
var tempDirSettings = new VoronPathSetting(tempDir);
Restore(env, package.Entries, tempDirSettings, toDispose, txw);
}
}
}
}
private List <TempBufferReport> GenerateTempBuffersReport(VoronPathSetting tempPath, VoronPathSetting journalPath)
{
var tempFiles = Directory.GetFiles(tempPath.FullPath, "*.buffers").Select(filePath =>
{
var file = new FileInfo(filePath);
return(new TempBufferReport
{
Name = file.Name,
AllocatedSpaceInBytes = file.Length,
Type = TempBufferType.Scratch
});
}).ToList();
if (journalPath != null)
{
var recyclableJournals = Directory.GetFiles(journalPath.FullPath, $"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}.*").Select(filePath =>
{
var file = new FileInfo(filePath);
return(new TempBufferReport
{
Name = file.Name,
AllocatedSpaceInBytes = file.Length,
Type = TempBufferType.RecyclableJournal
});
}).ToList();
tempFiles.AddRange(recyclableJournals);
}
return(tempFiles);
}
public void ToFile(StorageEnvironment env, VoronPathSetting backupPath,
CompressionLevel compression = CompressionLevel.Optimal,
Action <string> infoNotify = null,
Action backupStarted = null)
{
infoNotify = infoNotify ?? (s => { });
infoNotify("Voron backup db started");
using (var file = SafeFileStream.Create(backupPath.FullPath, FileMode.Create))
{
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
{
infoNotify("Voron backup started");
var dataPager = env.Options.DataPager;
var copier = new DataCopier(Constants.Storage.PageSize * 16);
Backup(env, compression, infoNotify, backupStarted, dataPager, package, string.Empty,
copier);
file.Flush(true); // make sure that we fully flushed to disk
}
}
infoNotify("Voron backup db finished");
}
public static StorageEnvironmentOptions ForPath(string path, string tempPath, string journalPath, IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification)
{
var pathSetting = new VoronPathSetting(path);
var tempPathSetting = new VoronPathSetting(tempPath ?? GetTempPath(path));
var journalPathSetting = journalPath != null ? new VoronPathSetting(journalPath) : null;
return(new DirectoryStorageEnvironmentOptions(pathSetting, tempPathSetting, journalPathSetting, ioChangesNotifications, catastrophicFailureNotification));
}
public Win32FileJournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long journalSize,
Win32NativeFileAccess access = Win32NativeFileAccess.GenericWrite,
Win32NativeFileShare shareMode = Win32NativeFileShare.Read)
{
try
{
_options = options;
_filename = filename;
_handle = Win32NativeFileMethods.CreateFile(filename.FullPath,
access, shareMode, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenAlways,
options.WinOpenFlags, IntPtr.Zero);
if (_handle.IsInvalid)
{
throw new IOException("When opening file " + filename, new Win32Exception(Marshal.GetLastWin32Error()));
}
var length = new FileInfo(filename.FullPath).Length;
if (length < journalSize)
{
try
{
Win32NativeFileMethods.SetFileLength(_handle, journalSize);
}
catch (Exception)
{
try
{
_handle?.Dispose();
_handle = null;
File.Delete(_filename.FullPath);
}
catch (Exception)
{
// there's nothing we can do about it
}
throw;
}
length = journalSize;
}
NumberOfAllocated4Kb = (int)(length / (4 * Constants.Size.Kilobyte));
_nativeOverlapped = (NativeOverlapped *)NativeMemory.AllocateMemory(sizeof(NativeOverlapped));
_nativeOverlapped->InternalLow = IntPtr.Zero;
_nativeOverlapped->InternalHigh = IntPtr.Zero;
}
catch
{
Dispose();
throw;
}
}
public void Restore(VoronPathSetting backupPath,
VoronPathSetting voronDataDir,
VoronPathSetting journalDir = null,
Action <string> onProgress = null,
CancellationToken cancellationToken = default)
{
using (var zip = ZipFile.Open(backupPath.FullPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
Restore(zip.Entries, voronDataDir, journalDir, onProgress, cancellationToken);
}
public PosixJournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long journalSize)
{
try
{
_options = options;
_filename = filename;
_maxNumberOf4KbPerSingleWrite = int.MaxValue / (4 * Constants.Size.Kilobyte);
_fd = Syscall.open(filename.FullPath, OpenFlags.O_WRONLY | options.PosixOpenFlags | PerPlatformValues.OpenFlags.O_CREAT,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
if (_fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when opening " + filename);
}
if (RunningOnMacOsx)
{
// mac doesn't support O_DIRECT, we fcntl instead:
var rc = Syscall.fcntl(_fd, FcntlCommands.F_NOCACHE, (IntPtr)1);
if (rc != 0)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when fcntl F_NOCACHE for " + filename);
}
}
var length = new FileInfo(filename.FullPath).Length;
if (length < journalSize)
{
length = journalSize;
try
{
PosixHelper.AllocateFileSpace(options, _fd, journalSize, filename.FullPath);
}
catch (Exception)
{
Syscall.close(_fd);
throw;
}
}
if (Syscall.CheckSyncDirectoryAllowed(_filename.FullPath) && Syscall.SyncDirectory(filename.FullPath) == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when syncing dir for on " + filename);
}
NumberOfAllocated4Kb = (int)(length / (4 * Constants.Size.Kilobyte));
}
catch
{
Dispose();
throw;
}
}
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path)
{
bool syncIsNeeded = false;
var fd = Syscall.open(path.FullPath, OpenFlags.O_WRONLY | PerPlatformValues.OpenFlags.O_CREAT,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
try
{
if (fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when opening " + path);
}
int remaining = sizeof(FileHeader);
FileInfo fi = new FileInfo(path.FullPath);
if (fi.Length != remaining)
{
syncIsNeeded = true;
}
var ptr = ((byte *)header);
while (remaining > 0)
{
var written = Syscall.write(fd, ptr, (ulong)remaining);
if (written == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "writing to " + path);
}
remaining -= (int)written;
ptr += written;
}
if (Syscall.FSync(fd) == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "FSync " + path);
}
if (syncIsNeeded)
{
Syscall.FsyncDirectoryFor(path.FullPath);
}
}
finally
{
if (fd != -1)
{
Syscall.close(fd);
fd = -1;
}
}
}
public Posix32BitsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null,
bool usePageProtection = false) : base(options, usePageProtection)
{
_options = options;
FileName = file;
if (Options.CopyOnWriteMode)
{
ThrowNotSupportedOption(file.FullPath);
}
_copyOnWriteMode = options.CopyOnWriteMode && file.FullPath.EndsWith(Constants.DatabaseFilename);
_isSyncDirAllowed = Syscall.CheckSyncDirectoryAllowed(FileName.FullPath);
PosixHelper.EnsurePathExists(FileName.FullPath);
Debug.Assert(RuntimeInformation.IsOSPlatform(OSPlatform.OSX) == false); // O_LARGEFILE not exists in mac and supported by default (however we do not run on mac 32bit..)
_fd = Syscall.open(file.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT | PerPlatformValues.OpenFlags.O_LARGEFILE,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
if (_fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when opening " + file);
}
_totalAllocationSize = GetFileSize();
if (_totalAllocationSize == 0 && initialFileSize.HasValue)
{
_totalAllocationSize = NearestSizeToAllocationGranularity(initialFileSize.Value);
}
if (_totalAllocationSize == 0 || _totalAllocationSize % AllocationGranularity != 0 ||
_totalAllocationSize != GetFileSize())
{
_totalAllocationSize = NearestSizeToAllocationGranularity(_totalAllocationSize);
PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, file.FullPath);
}
if (_isSyncDirAllowed && Syscall.SyncDirectory(file.FullPath) == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "sync dir for " + file);
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
SetPagerState(new PagerState(this)
{
Files = null,
MapBase = null,
AllocationInfos = new PagerState.AllocationInfo[0]
});
}
private List <TempBufferReport> GenerateTempBuffersReport(VoronPathSetting tempPath)
{
return(Directory.GetFiles(tempPath.FullPath, "*.buffers").Select(filePath =>
{
var file = new FileInfo(filePath);
return new TempBufferReport
{
Name = file.Name,
AllocatedSpaceInBytes = file.Length
};
}).ToList());
}
public void FullBackupMustNotDeadlockWithFlush()
{
RequireFileBasedPager();
var random = new Random(2);
var buffer = new byte[8192];
random.NextBytes(buffer);
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
for (int i = 0; i < 5000; i++)
{
tree.Add("items/" + i, new MemoryStream(buffer));
}
tx.Commit();
}
var voronDataDir = new VoronPathSetting(DataDir);
var backupCompleted = false;
Exception backupException = null;
Thread backup = new Thread(() =>
{
try
{
BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup"));
backupCompleted = true;
}
catch (Exception e)
{
backupException = e;
}
});
Env.Journal.Applicator.ForTestingPurposesOnly().OnApplyLogsToDataFileUnderFlushingLock += () =>
{
backup.Start();
};
Env.FlushLogToDataFile();
backup.Join(TimeSpan.FromSeconds(60));
Assert.Null(backupException);
Assert.True(backupCompleted);
}
public PosixMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null,
bool usePageProtection = false) : base(options, canPrefetchAhead: true, usePageProtection: usePageProtection)
{
_options = options;
FileName = file;
_copyOnWriteMode = options.CopyOnWriteMode && file.FullPath.EndsWith(Constants.DatabaseFilename);
_isSyncDirAllowed = Syscall.CheckSyncDirectoryAllowed(FileName.FullPath);
PosixHelper.EnsurePathExists(FileName.FullPath);
_fd = Syscall.open(file.FullPath, OpenFlags.O_RDWR | PerPlatformValues.OpenFlags.O_CREAT,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
if (_fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when opening " + file);
}
SysPageSize = Syscall.sysconf(PerPlatformValues.SysconfNames._SC_PAGESIZE);
if (SysPageSize <= 0) // i.e. -1 because _SC_PAGESIZE defined differently on various platforms
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "Got SysPageSize <= 0 for " + FileName);
}
_totalAllocationSize = GetFileSize();
if (_totalAllocationSize == 0 && initialFileSize.HasValue)
{
_totalAllocationSize = NearestSizeToPageSize(initialFileSize.Value);
}
if (_totalAllocationSize == 0 || _totalAllocationSize % SysPageSize != 0 ||
_totalAllocationSize != GetFileSize())
{
_totalAllocationSize = NearestSizeToPageSize(_totalAllocationSize);
PosixHelper.AllocateFileSpace(_options, _fd, _totalAllocationSize, file.FullPath);
}
if (_isSyncDirAllowed && Syscall.SyncDirectory(file.FullPath) == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "sync dir for " + file);
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
SetPagerState(CreatePagerState());
}
public void Can_reuse_files_for_cache()
{
var path = new VoronPathSetting(NewDataPath());
var environment = new StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions(path, path, path, null, null);
using (File.Create(TempFileCache.GetTempFileName(environment)))
{
}
using (var cache = new TempFileCache(environment))
{
Assert.Equal(1, cache.FilesCount);
}
}
public JournalWriter(StorageEnvironmentOptions options, VoronPathSetting filename, long size, PalFlags.JournalMode mode = PalFlags.JournalMode.Safe)
{
_options = options;
FileName = filename;
_log = LoggingSource.Instance.GetLogger <JournalWriter>(options.BasePath.FullPath);
var result = Pal.rvn_open_journal_for_writes(filename.FullPath, mode, size, options.SupportDurabilityFlags, out _writeHandle, out var actualSize, out var error);
if (result != PalFlags.FailCodes.Success)
{
PalHelper.ThrowLastError(result, error, $"Attempted to open journal file - Path: {filename.FullPath} Size :{size}");
}
NumberOfAllocated4Kb = (int)(actualSize / (4 * Constants.Size.Kilobyte));
}
private void EnsureMinimumSize(FileInfo fileInfo, VoronPathSetting path)
{
try
{
using (var stream = fileInfo.Open(FileMode.OpenOrCreate))
{
stream.SetLength(InitialLogFileSize);
}
}
catch (Exception e)
{
throw new InvalidOperationException(
"Journal file " + path + " could not be opened because it's size is too small and we couldn't increase it",
e);
}
}
protected StorageEnvironmentOptions(VoronPathSetting tempPath, IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification)
{
SafePosixOpenFlags = SafePosixOpenFlags | DefaultPosixFlags;
DisposeWaitTime = TimeSpan.FromSeconds(15);
TempPath = tempPath;
ShouldUseKeyPrefix = name => false;
MaxLogFileSize = ((sizeof(int) == IntPtr.Size ? 32 : 256) * Constants.Size.Megabyte);
InitialLogFileSize = 64 * Constants.Size.Kilobyte;
MaxScratchBufferSize = ((sizeof(int) == IntPtr.Size ? 32 : 256) * Constants.Size.Megabyte);
MaxNumberOfPagesInJournalBeforeFlush =
((sizeof(int) == IntPtr.Size ? 4 : 32) * Constants.Size.Megabyte) / Constants.Storage.PageSize;
IdleFlushTimeout = 5000; // 5 seconds
OwnsPagers = true;
IncrementalBackupEnabled = false;
IoMetrics = new IoMetrics(256, 256, ioChangesNotifications);
_log = LoggingSource.Instance.GetLogger <StorageEnvironment>(tempPath.FullPath);
_catastrophicFailureNotification = catastrophicFailureNotification ?? new CatastrophicFailureNotification((e) =>
{
if (_log.IsOperationsEnabled)
{
_log.Operations($"Catastrophic failure in {this}", e);
}
});
var shouldForceEnvVar = Environment.GetEnvironmentVariable("VORON_INTERNAL_ForceUsing32BitsPager");
bool result;
if (bool.TryParse(shouldForceEnvVar, out result))
{
ForceUsing32BitsPager = result;
}
}
private AbstractPager OpenPagerInternal(VoronPathSetting filename)
{
if (RunningOnPosix)
{
if (RunningOn32Bits)
{
return(new Posix32BitsMemoryMapPager(this, filename));
}
return(new PosixMemoryMapPager(this, filename));
}
if (RunningOn32Bits)
{
return(new Windows32BitsMemoryMapPager(this, filename));
}
return(new WindowsMemoryMapPager(this, filename));
}
private void Restore(StorageEnvironment env, IEnumerable <ZipArchiveEntry> entries)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
env.FlushLogToDataFile();
var transactionPersistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite))
{
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
var tempDirSettings = new VoronPathSetting(tempDir);
Restore(env, entries, tempDirSettings, toDispose, txw);
}
}
}
public void Restore(IEnumerable <ZipArchiveEntry> entries,
VoronPathSetting voronDataDir,
VoronPathSetting journalDir = null,
Action <string> onProgress = null,
CancellationToken cancellationToken = default)
{
journalDir = journalDir ?? voronDataDir.Combine("Journals");
if (Directory.Exists(voronDataDir.FullPath) == false)
{
Directory.CreateDirectory(voronDataDir.FullPath);
}
if (Directory.Exists(journalDir.FullPath) == false)
{
Directory.CreateDirectory(journalDir.FullPath);
}
onProgress?.Invoke("Starting snapshot restore");
foreach (var entry in entries)
{
var dst = string.Equals(Path.GetExtension(entry.Name), ".journal", StringComparison.OrdinalIgnoreCase)
? journalDir
: voronDataDir;
var sw = Stopwatch.StartNew();
if (Directory.Exists(dst.FullPath) == false)
{
Directory.CreateDirectory(dst.FullPath);
}
using (var input = entry.Open())
using (var output = SafeFileStream.Create(dst.Combine(entry.Name).FullPath, FileMode.CreateNew))
{
input.CopyTo(output, cancellationToken);
}
onProgress?.Invoke($"Restored file: '{entry.Name}' to: '{dst}', " +
$"size in bytes: {entry.Length:#,#;;0}, " +
$"took: {sw.ElapsedMilliseconds:#,#;;0}ms");
}
}
public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path)
{
var fd = Syscall.open(path.FullPath, OpenFlags.O_RDONLY, FilePermissions.S_IRUSR);
try
{
if (fd == -1)
{
var lastError = Marshal.GetLastWin32Error();
if (((Errno)lastError) == Errno.EACCES)
{
return(false);
}
Syscall.ThrowLastError(lastError);
}
int remaining = sizeof(FileHeader);
var ptr = ((byte *)header);
while (remaining > 0)
{
var read = Syscall.read(fd, ptr, (ulong)remaining);
if (read == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err);
}
if (read == 0)
{
return(false); // truncated file?
}
remaining -= (int)read;
ptr += read;
}
return(true);
}
finally
{
if (fd != -1)
{
Syscall.close(fd);
fd = -1;
}
}
}
public PureMemoryStorageEnvironmentOptions(string name, VoronPathSetting tempPath,
IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification)
: base(tempPath, ioChangesNotifications, catastrophicFailureNotification)
{
_name = name;
_instanceId = Interlocked.Increment(ref _counter);
var guid = Guid.NewGuid();
var filename = $"ravendb-{Process.GetCurrentProcess().Id}-{_instanceId}-data.pager-{guid}";
WinOpenFlags = Win32NativeFileAttributes.Temporary | Win32NativeFileAttributes.DeleteOnClose;
if (Directory.Exists(tempPath.FullPath) == false)
{
Directory.CreateDirectory(tempPath.FullPath);
}
_dataPager = new Lazy <AbstractPager>(() => GetTempMemoryMapPager(this, TempPath.Combine(filename), InitialFileSize,
Win32NativeFileAttributes.RandomAccess | Win32NativeFileAttributes.DeleteOnClose | Win32NativeFileAttributes.Temporary), true);
}
public void Skip_files_that_are_in_use()
{
var path = new VoronPathSetting(NewDataPath());
var environment = new StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions(path, path, path, null, null);
for (var i = 0; i < TempFileCache.MaxFilesToKeepInCache; i++)
{
using (File.Create(TempFileCache.GetTempFileName(environment)))
{
}
}
using (File.Create(Path.Combine(environment.TempPath.FullPath,
TempFileCache.FilePrefix + "Z" + StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions.TempFileExtension)))
{
using (new TempFileCache(environment))
{
}
}
}
private static bool TryMoveJournals(VoronPathSetting basePath, VoronPathSetting journalsPath)
{
var journalsInRoot = Directory.GetFiles(basePath.FullPath, "*.journal", SearchOption.TopDirectoryOnly);
if (journalsInRoot.Length == 0)
{
return(false);
}
foreach (var journalFile in journalsInRoot)
{
var journalFileInfo = new FileInfo(journalFile);
var source = basePath.Combine(journalFileInfo.Name);
var destination = journalsPath.Combine(journalFileInfo.Name);
File.Move(source.FullPath, destination.FullPath);
}
return(true);
}
private static void BackupAndDeleteFile(VoronPathSetting path)
{
if (File.Exists(path.FullPath) == false)
{
return;
}
var count = 0;
while (true)
{
var filePath = $"{path.FullPath}.{count++}.bak";
if (File.Exists(filePath))
{
continue;
}
File.Move(path.FullPath, filePath);
break;
}
}
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path)
{
using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
{
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int written;
if (Win32NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out written, null) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to write to file " + path);
}
ptr += written;
remaining -= written;
}
if (Win32NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to Flush File Buffers (sync) of file " + path);
}
}
}
public DirectoryStorageEnvironmentOptions(VoronPathSetting basePath, VoronPathSetting tempPath, VoronPathSetting journalPath,
IoChangesNotifications ioChangesNotifications, CatastrophicFailureNotification catastrophicFailureNotification)
: base(tempPath ?? basePath, ioChangesNotifications, catastrophicFailureNotification)
{
_basePath = basePath;
_journalPath = journalPath ?? basePath;
if (Directory.Exists(_basePath.FullPath) == false)
{
Directory.CreateDirectory(_basePath.FullPath);
}
if (Equals(_basePath, tempPath) == false && Directory.Exists(TempPath.FullPath) == false)
{
Directory.CreateDirectory(TempPath.FullPath);
}
if (Equals(_journalPath, tempPath) == false && Directory.Exists(_journalPath.FullPath) == false)
{
Directory.CreateDirectory(_journalPath.FullPath);
}
_dataPager = new Lazy <AbstractPager>(() =>
{
FilePath = _basePath.Combine(Constants.DatabaseFilename);
return(GetMemoryMapPager(this, InitialFileSize, FilePath, usePageProtection: true));
});
// have to be before the journal check, so we'll fail on files in use
// TODO: Need to verify behavior on Linux, might need to maintain a file lock
DeleteAllTempBuffers();
GatherRecyclableJournalFiles(); // if there are any (e.g. after a rude db shut down) let us reuse them
}
public WindowsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file,
long?initialFileSize = null,
Win32NativeFileAttributes fileAttributes = Win32NativeFileAttributes.Normal,
Win32NativeFileAccess access = Win32NativeFileAccess.GenericRead | Win32NativeFileAccess.GenericWrite,
bool usePageProtection = false)
: base(options, !fileAttributes.HasFlag(Win32NativeFileAttributes.Temporary), usePageProtection)
{
SYSTEM_INFO systemInfo;
GetSystemInfo(out systemInfo);
FileName = file;
_logger = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}");
_access = access;
_copyOnWriteMode = Options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename);
if (_copyOnWriteMode)
{
_memoryMappedFileAccess = MemoryMappedFileAccess.Read | MemoryMappedFileAccess.CopyOnWrite;
fileAttributes = Win32NativeFileAttributes.Readonly;
_access = Win32NativeFileAccess.GenericRead;
}
else
{
_memoryMappedFileAccess = _access == Win32NativeFileAccess.GenericRead
? MemoryMappedFileAccess.Read
: MemoryMappedFileAccess.ReadWrite;
}
_fileAttributes = fileAttributes;
_handle = Win32NativeFileMethods.CreateFile(file.FullPath, access,
Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenAlways, fileAttributes, IntPtr.Zero);
if (_handle.IsInvalid)
{
int lastWin32ErrorCode = Marshal.GetLastWin32Error();
throw new IOException("Failed to open file storage of WinMemoryMapPager for " + file,
new Win32Exception(lastWin32ErrorCode));
}
_fileInfo = new FileInfo(file.FullPath);
var drive = _fileInfo.Directory.Root.Name.TrimEnd('\\');
try
{
if (PhysicalDrivePerMountCache.TryGetValue(drive, out UniquePhysicalDriveId) == false)
{
UniquePhysicalDriveId = GetPhysicalDriveId(drive);
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Physical drive '{drive}' unique id = '{UniquePhysicalDriveId}' for file '{file}'");
}
}
catch (Exception ex)
{
UniquePhysicalDriveId = 0;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to determine physical drive Id for drive letter '{drive}', file='{file}'", ex);
}
}
var streamAccessType = _access == Win32NativeFileAccess.GenericRead
? FileAccess.Read
: FileAccess.ReadWrite;
_fileStream = SafeFileStream.Create(_handle, streamAccessType);
_totalAllocationSize = _fileInfo.Length;
if ((access & Win32NativeFileAccess.GenericWrite) == Win32NativeFileAccess.GenericWrite ||
(access & Win32NativeFileAccess.GenericAll) == Win32NativeFileAccess.GenericAll ||
(access & Win32NativeFileAccess.FILE_GENERIC_WRITE) == Win32NativeFileAccess.FILE_GENERIC_WRITE)
{
var fileLength = _fileStream.Length;
if (fileLength == 0 && initialFileSize.HasValue)
{
fileLength = initialFileSize.Value;
}
if (_fileStream.Length == 0 || (fileLength % AllocationGranularity != 0))
{
fileLength = NearestSizeToAllocationGranularity(fileLength);
Win32NativeFileMethods.SetFileLength(_handle, fileLength);
}
_totalAllocationSize = fileLength;
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
SetPagerState(CreatePagerState());
}
protected async Task <RestoreSettings> SnapshotRestore(JsonOperationContext context, string backupPath,
Action <IOperationProgress> onProgress, RestoreResult restoreResult)
{
Debug.Assert(onProgress != null);
RestoreSettings restoreSettings = null;
var fullBackupPath = GetBackupPath(backupPath);
using (var zip = await GetZipArchiveForSnapshot(fullBackupPath))
{
foreach (var zipEntries in zip.Entries.GroupBy(x => x.FullName.Substring(0, x.FullName.Length - x.Name.Length)))
{
var directory = zipEntries.Key;
if (string.IsNullOrWhiteSpace(directory))
{
foreach (var zipEntry in zipEntries)
{
if (string.Equals(zipEntry.Name, RestoreSettings.SettingsFileName, StringComparison.OrdinalIgnoreCase))
{
using (var entryStream = zipEntry.Open())
{
var snapshotEncryptionKey = RestoreFromConfiguration.EncryptionKey != null
? Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey)
: null;
using (var stream = GetInputStream(entryStream, snapshotEncryptionKey))
{
var json = context.Read(stream, "read database settings for restore");
json.BlittableValidation();
restoreSettings = JsonDeserializationServer.RestoreSettings(json);
restoreSettings.DatabaseRecord.DatabaseName = RestoreFromConfiguration.DatabaseName;
DatabaseHelper.Validate(RestoreFromConfiguration.DatabaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
if (restoreSettings.DatabaseRecord.Encrypted && _hasEncryptionKey == false)
{
throw new ArgumentException("Database snapshot is encrypted but the encryption key is missing!");
}
if (restoreSettings.DatabaseRecord.Encrypted == false && _hasEncryptionKey)
{
throw new ArgumentException("Cannot encrypt a non encrypted snapshot backup during restore!");
}
}
}
}
}
continue;
}
var voronDataDirectory = new VoronPathSetting(RestoreFromConfiguration.DataDirectory);
var restoreDirectory = directory.StartsWith(Constants.Documents.PeriodicBackup.Folders.Documents, StringComparison.OrdinalIgnoreCase)
? voronDataDirectory
: voronDataDirectory.Combine(directory);
BackupMethods.Full.Restore(
zipEntries,
restoreDirectory,
journalDir: null,
onProgress: message =>
{
restoreResult.AddInfo(message);
restoreResult.SnapshotRestore.ReadCount++;
onProgress.Invoke(restoreResult.Progress);
},
cancellationToken: _operationCancelToken.Token);
}
}
if (restoreSettings == null)
{
throw new InvalidDataException("Cannot restore the snapshot without the settings file!");
}
return(restoreSettings);
}
