public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path)
{
using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
{
if (fs.Length != sizeof(FileHeader))
{
return(false); // wrong file size
}
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (Win32NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to read file " + path);
}
if (read == 0)
{
return(false); // we should be reading _something_ here, if we can't, then it is an error and we assume corruption
}
ptr += read;
remaining -= read;
}
return(true);
}
}
public HeaderAccessor(StorageEnvironment env)
{
_env = env;
_headerPtr = Marshal.AllocHGlobal(sizeof(FileHeader));
_theHeader = (FileHeader *)_headerPtr.ToPointer();
}
public static unsafe bool TryReadFileHeader(FileHeader *header, string path)
{
using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
if (fs.Length != sizeof(FileHeader))
{
return(false); // wrong file size
}
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (Win32NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception();
}
if (read == 0)
{
return(false); // we should be reading _something_ here, if we can't, then it is an error and we assume corruption
}
ptr += read;
remaining -= read;
}
return(true);
}
}
public HeaderAccessor(StorageEnvironment env)
{
_env = env;
_headerPtr = NativeMemory.AllocateMemory(sizeof(FileHeader));
_theHeader = (FileHeader *)_headerPtr;
}
public unsafe override bool ReadHeader(string filename, FileHeader *header)
{
var path = Path.Combine(_basePath, filename);
if (File.Exists(path) == false)
{
return(false);
}
using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception();
}
ptr += read;
remaining -= read;
}
return(true);
}
}
public static unsafe void WriteFileHeader(FileHeader *header, string path)
{
var fd = Syscall.open(path, OpenFlags.O_WRONLY | OpenFlags.O_CREAT,
FilePermissions.ALLPERMS);
try
{
if (fd == -1)
{
ThrowLastError(Marshal.GetLastWin32Error());
}
int remaining = sizeof(FileHeader);
var ptr = ((byte *)header);
while (remaining > 0)
{
var written = Syscall.write(fd, ptr, (ulong)remaining);
if (written == -1)
{
ThrowLastError(Marshal.GetLastWin32Error());
}
remaining -= (int)written;
ptr += written;
}
Syscall.fsync(fd);
}
finally
{
if (fd != -1)
{
Syscall.close(fd);
}
}
}
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path)
{
bool syncIsNeeded = false;
var fd = Syscall.open(path.FullPath, OpenFlags.O_WRONLY | PerPlatformValues.OpenFlags.O_CREAT,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
try
{
if (fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "when opening " + path);
}
int remaining = sizeof(FileHeader);
FileInfo fi = new FileInfo(path.FullPath);
if (fi.Length != remaining)
{
syncIsNeeded = true;
}
var ptr = ((byte *)header);
while (remaining > 0)
{
var written = Syscall.write(fd, ptr, (ulong)remaining);
if (written == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "writing to " + path);
}
remaining -= (int)written;
ptr += written;
}
if (Syscall.FSync(fd) == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, "FSync " + path);
}
if (syncIsNeeded)
{
Syscall.FsyncDirectoryFor(path.FullPath);
}
}
finally
{
if (fd != -1)
{
Syscall.close(fd);
fd = -1;
}
}
}
public override unsafe void WriteHeader(string filename, FileHeader *header)
{
IntPtr ptr;
if (_headers.TryGetValue(filename, out ptr) == false)
{
ptr = Marshal.AllocHGlobal(sizeof(FileHeader));
_headers[filename] = ptr;
}
NativeMethods.memcpy((byte *)ptr, (byte *)header, sizeof(FileHeader));
}
public unsafe override bool ReadHeader(string filename, FileHeader *header)
{
IntPtr ptr;
if (_headers.TryGetValue(filename, out ptr) == false)
{
return(false);
}
*header = *((FileHeader *)ptr);
return(true);
}
private static unsafe void WriteFileHeader(FileStream fs, FileHeader header)
{
byte[] data = new byte[sizeof(FileHeader)];
fixed(byte *pData = data)
{
FileHeader *temp = (FileHeader *)pData;
*temp = header;
}
fs.Write(data, 0, sizeof(FileHeader));
}
public override unsafe void WriteHeader(string filename, FileHeader *header)
{
var path = Path.Combine(_basePath, filename);
if (RunningOnPosix)
{
PosixHelper.WriteFileHeader(header, path);
}
else
{
Win32Helper.WriteFileHeader(header, path);
}
}
public unsafe override bool ReadHeader(string filename, FileHeader *header)
{
var path = Path.Combine(_basePath, filename);
if (File.Exists(path) == false)
{
return(false);
}
if (RunningOnPosix)
{
return(PosixHelper.TryReadFileHeader(header, path));
}
return(Win32Helper.TryReadFileHeader(header, path));
}
public unsafe override bool ReadHeader(string filename, FileHeader *header)
{
if (Disposed)
{
throw new ObjectDisposedException("PureMemoryStorageEnvironmentOptions");
}
IntPtr ptr;
if (_headers.TryGetValue(filename, out ptr) == false)
{
return(false);
}
*header = *((FileHeader *)ptr);
return(true);
}
public override unsafe bool ReadHeader(string filename, FileHeader *header)
{
if (Disposed)
{
throw new ObjectDisposedException("PureMemoryStorageEnvironmentOptions");
}
IntPtr ptr;
if (_headers.TryGetValue(filename, out ptr) == false)
{
return(false);
}
*header = *((FileHeader *)ptr);
return(header->Hash == HeaderAccessor.CalculateFileHeaderHash(header));
}
private bool IsEmptyHeader(FileHeader *header)
{
return(header->MagicMarker == Constants.MagicMarker &&
header->Version == Constants.CurrentVersion &&
header->HeaderRevision == -1 &&
header->TransactionId == 0 &&
header->LastPageNumber == 1 &&
header->Root.RootPageNumber == -1 &&
header->Journal.CurrentJournal == -1 &&
header->Journal.JournalFilesCount == 0 &&
header->Journal.LastSyncedJournal == -1 &&
header->Journal.LastSyncedTransactionId == -1 &&
header->IncrementalBackup.LastBackedUpJournal == -1 &&
header->IncrementalBackup.LastBackedUpJournalPage == -1 &&
header->IncrementalBackup.LastCreatedJournal == -1);
}
private FileHeader *FindLatestFileHeadeEntry()
{
Page fst = _pager.Get(null, 0);
Page snd = _pager.Get(null, 1);
FileHeader *e1 = GetFileHeaderFrom(fst);
FileHeader *e2 = GetFileHeaderFrom(snd);
FileHeader *entry = e1;
if (e2->TransactionId > e1->TransactionId)
{
entry = e2;
}
return(entry);
}
public override unsafe void WriteHeader(string filename, FileHeader *header)
{
if (Disposed)
{
throw new ObjectDisposedException("PureMemoryStorageEnvironmentOptions");
}
IntPtr ptr;
if (_headers.TryGetValue(filename, out ptr) == false)
{
ptr = Marshal.AllocHGlobal(sizeof(FileHeader));
_headers[filename] = ptr;
}
MemoryUtils.Copy((byte *)ptr, (byte *)header, sizeof(FileHeader));
}
public static unsafe void WriteFileHeader(FileHeader *header, string path)
{
var fd = Syscall.open(path, OpenFlags.O_WRONLY | OpenFlags.O_CREAT,
FilePermissions.S_IWUSR | FilePermissions.S_IRUSR);
try
{
if (fd == -1)
{
var err = Marshal.GetLastWin32Error();
ThrowLastError(err, "when opening " + path);
}
int remaining = sizeof(FileHeader);
var ptr = ((byte *)header);
while (remaining > 0)
{
var written = Syscall.write(fd, ptr, (ulong)remaining);
if (written == -1)
{
var err = Marshal.GetLastWin32Error();
ThrowLastError(err, "writing to " + path);
}
remaining -= (int)written;
ptr += written;
}
if (Syscall.fsync(fd) == -1)
{
var err = Marshal.GetLastWin32Error();
ThrowLastError(err, "fsync " + path);
}
if (CheckSyncDirectoryAllowed(path) && SyncDirectory(path) == -1)
{
var err = Marshal.GetLastWin32Error();
ThrowLastError(err, "fsync dir " + path);
}
}
finally
{
if (fd != -1)
{
Syscall.close(fd);
fd = -1;
}
}
}
public void Dispose()
{
_locker.EnterWriteLock();
try
{
if (_headerPtr != IntPtr.Zero)
{
Marshal.FreeHGlobal(_headerPtr);
_headerPtr = IntPtr.Zero;
_theHeader = null;
}
}
finally
{
_locker.ExitWriteLock();
}
}
public void Dispose()
{
_locker.EnterWriteLock();
try
{
if (_headerPtr != null)
{
NativeMemory.Free(_headerPtr, sizeof(FileHeader));
_headerPtr = null;
_theHeader = null;
}
}
finally
{
_locker.ExitWriteLock();
}
}
private static void FillInEmptyHeader(FileHeader *header)
{
header->MagicMarker = Constants.MagicMarker;
header->Version = Constants.CurrentVersion;
header->HeaderRevision = -1;
header->TransactionId = 0;
header->LastPageNumber = 1;
header->FreeSpace.RootPageNumber = -1;
header->Root.RootPageNumber = -1;
header->Journal.CurrentJournal = -1;
header->Journal.JournalFilesCount = 0;
header->Journal.LastSyncedJournal = -1;
header->Journal.LastSyncedTransactionId = -1;
header->IncrementalBackup.LastBackedUpJournal = -1;
header->IncrementalBackup.LastBackedUpJournalPage = -1;
header->IncrementalBackup.LastCreatedJournal = -1;
}
public static ulong CalculateFileHeaderHash(FileHeader *header)
{
var ctx = Hashing.Streamed.XXHash64.BeginProcess((ulong)header->TransactionId);
// First part of header, until the Hash field
Hashing.Streamed.XXHash64.Process(ctx, (byte *)header, FileHeader.HashOffset);
// Second part of header, after the hash field
var secondPartOfHeaderLength = sizeof(FileHeader) - (FileHeader.HashOffset + sizeof(ulong));
if (secondPartOfHeaderLength > 0)
{
Hashing.Streamed.XXHash64.Process(ctx, (byte *)header + FileHeader.HashOffset + sizeof(ulong), secondPartOfHeaderLength);
}
return(Hashing.Streamed.XXHash64.EndProcess(ctx));
}
private void FillInEmptyHeader(FileHeader *header)
{
header->MagicMarker = Constants.MagicMarker;
header->Version = Constants.CurrentVersion;
header->HeaderRevision = -1;
header->TransactionId = 0;
header->LastPageNumber = 1;
header->Root.RootPageNumber = -1;
header->Journal.CurrentJournal = -1;
Memory.Set(header->Journal.Reserved, 0, JournalInfo.NumberOfReservedBytes);
header->Journal.Flags = Journal.JournalInfoFlags.None;
header->Journal.LastSyncedJournal = -1;
header->Journal.LastSyncedTransactionId = -1;
header->IncrementalBackup.LastBackedUpJournal = -1;
header->IncrementalBackup.LastBackedUpJournalPage = -1;
header->IncrementalBackup.LastCreatedJournal = -1;
header->PageSize = _env.Options.PageSize;
}
public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path)
{
var fd = Syscall.open(path.FullPath, OpenFlags.O_RDONLY, FilePermissions.S_IRUSR);
try
{
if (fd == -1)
{
var lastError = Marshal.GetLastWin32Error();
if (((Errno)lastError) == Errno.EACCES)
{
return(false);
}
Syscall.ThrowLastError(lastError);
}
int remaining = sizeof(FileHeader);
var ptr = ((byte *)header);
while (remaining > 0)
{
var read = Syscall.read(fd, ptr, (ulong)remaining);
if (read == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err);
}
if (read == 0)
{
return(false); // truncated file?
}
remaining -= (int)read;
ptr += read;
}
return(true);
}
finally
{
if (fd != -1)
{
Syscall.close(fd);
fd = -1;
}
}
}
public static unsafe void WriteFileHeader(FileHeader *header, string path)
{
using (var fs = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.Read))
{
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (Win32NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception();
}
ptr += read;
remaining -= read;
}
Win32NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle);
}
}
private bool IsEmptyHeader(FileHeader *header)
{
var zeroed = stackalloc byte[JournalInfo.NumberOfReservedBytes];
return(header->MagicMarker == Constants.MagicMarker &&
header->Version == Constants.CurrentVersion &&
header->HeaderRevision == -1 &&
header->TransactionId == 0 &&
header->LastPageNumber == 1 &&
header->Root.RootPageNumber == -1 &&
header->Journal.CurrentJournal == -1 &&
header->Journal.Flags == Journal.JournalInfoFlags.None &&
Memory.Compare(header->Journal.Reserved, zeroed, 3) == 0 &&
header->Journal.LastSyncedJournal == -1 &&
header->Journal.LastSyncedTransactionId == -1 &&
header->IncrementalBackup.LastBackedUpJournal == -1 &&
header->IncrementalBackup.LastBackedUpJournalPage == -1 &&
header->IncrementalBackup.LastCreatedJournal == -1);
}
private void Setup(IVirtualPager pager)
{
if (pager.NumberOfAllocatedPages == 0)
{
WriteEmptyHeaderPage(_pager.Get(null, 0));
WriteEmptyHeaderPage(_pager.Get(null, 1));
NextPageNumber = 2;
using (var tx = new Transaction(_pager, this, _transactionsCounter + 1, TransactionFlags.ReadWrite, _freeSpaceRepository))
{
var root = Tree.Create(tx, _sliceComparer);
var freeSpace = Tree.Create(tx, _sliceComparer);
// important to first create the two trees, then set them on the env
FreeSpaceRoot = freeSpace;
Root = root;
tx.UpdateRoots(root, freeSpace);
tx.Commit();
}
return;
}
// existing db, let us load it
// the first two pages are allocated for double buffering tx commits
FileHeader *entry = FindLatestFileHeadeEntry();
NextPageNumber = entry->LastPageNumber + 1;
_transactionsCounter = entry->TransactionId + 1;
using (var tx = new Transaction(_pager, this, _transactionsCounter + 1, TransactionFlags.ReadWrite, _freeSpaceRepository))
{
var root = Tree.Open(tx, _sliceComparer, &entry->Root);
var freeSpace = Tree.Open(tx, _sliceComparer, &entry->FreeSpace);
// important to first create the two trees, then set them on the env
FreeSpaceRoot = freeSpace;
Root = root;
tx.Commit();
}
}
public override unsafe bool ReadHeader(string filename, FileHeader *header)
{
var path = _basePath.Combine(filename);
if (File.Exists(path.FullPath) == false)
{
return(false);
}
var success = RunningOnPosix ?
PosixHelper.TryReadFileHeader(header, path) :
Win32Helper.TryReadFileHeader(header, path);
if (!success)
{
return(false);
}
return(header->Hash == HeaderAccessor.CalculateFileHeaderHash(header));
}
public override unsafe void WriteHeader(string filename, FileHeader *header)
{
var path = Path.Combine(_basePath, filename);
using (var fs = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.Read))
{
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception();
}
ptr += read;
remaining -= read;
}
NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle);
}
}
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path)
{
using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
{
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int written;
if (Win32NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out written, null) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to write to file " + path);
}
ptr += written;
remaining -= written;
}
if (Win32NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to Flush File Buffers (sync) of file " + path);
}
}
}
