public bool TryGetValue(Transaction tx, long page, out JournalFile.PagePosition value)
{
ImmutableAppendOnlyList<PageValue> list;
if (_values.TryGetValue(page, out list) == false)
{
value = null;
return false;
}
for (int i = list.Count - 1; i >= 0; i--)
{
var it = list[i];
if (it.Transaction > tx.Id)
continue;
if (it.Value.IsFreedPageMarker)
break;
value = it.Value;
Debug.Assert(value != null);
return true;
}
// all the current values are _after_ this transaction started, so it sees nothing
value = null;
return false;
}
private JournalFile NextFile(int numberOfPages = 1)
{
_journalIndex++;
var now = DateTime.UtcNow;
if ((now - _lastFile).TotalSeconds < 90)
{
_currentJournalFileSize = Math.Min(_env.Options.MaxLogFileSize, _currentJournalFileSize * 2);
}
var actualLogSize = _currentJournalFileSize;
var minRequiredSize = numberOfPages * AbstractPager.PageSize;
if (_currentJournalFileSize < minRequiredSize)
{
actualLogSize = minRequiredSize;
}
_lastFile = now;
var journalPager = _env.Options.CreateJournalWriter(_journalIndex, actualLogSize);
var journal = new JournalFile(journalPager, _journalIndex);
journal.AddRef(); // one reference added by a creator - write ahead log
_files = _files.Append(journal);
_headerAccessor.Modify(_updateLogInfo);
return journal;
}
public void WriteBufferToFile(JournalFile journalFile, LowLevelTransaction tx)
{
if (_firstPositionInJournalFile != null)
{
journalFile.WriteBuffer(_firstPositionInJournalFile.Value, _lazyTransactionPager.AcquirePagePointer(null, 0),
_lastUsedPage * _options.DataPager.PageSize);
}
if (tx != null)
{
tx.IsLazyTransaction = false;// so it will notify the flush thread it has work to do
}
_readTransaction?.Dispose();
_firstPositionInJournalFile = null;
_lastUsedPage = 0;
_readTransaction = null;
NumberOfPages = 0;
}
public void ToStream(JournalFile journal, long startPage, long pagesToCopy, Stream output)
{
var maxNumOfPagesToCopyAtOnce = _buffer.Length/AbstractPager.PageSize;
var page = startPage;
fixed (byte* ptr = _buffer)
{
while (pagesToCopy > 0)
{
var pageCount = Math.Min(maxNumOfPagesToCopyAtOnce, pagesToCopy);
var bytesCount = (int) (pageCount*AbstractPager.PageSize);
journal.JournalWriter.Read(page, ptr, bytesCount);
output.Write(_buffer, 0, bytesCount);
page += pageCount;
pagesToCopy -= pageCount;
}
}
Debug.Assert(pagesToCopy == 0);
}
public void WriteToJournal(Transaction tx, int pageCount)
{
var pages = CompressPages(tx, pageCount, _compressionPager);
if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length)
{
CurrentFile = NextFile(pages.Length);
}
var transactionHeader = *(TransactionHeader *)pages[0];
var writePage = CurrentFile.Write(tx, pages);
var onTransactionCommit = OnTransactionCommit;
if (onTransactionCommit != null)
{
var bufferSize = pages.Length * AbstractPager.PageSize;
var buffer = new byte[bufferSize];
fixed(byte *bp = buffer)
CurrentFile.JournalWriter.Read(writePage, bp, bufferSize);
var stream = new MemoryStream(buffer, AbstractPager.PageSize, (pages.Length - 1) * AbstractPager.PageSize);
var transactionToShip = new TransactionToShip(transactionHeader)
{
CompressedData = stream,
PreviousTransactionCrc = _previousTransactionCrc
};
_previousTransactionCrc = transactionHeader.Crc;
onTransactionCommit(transactionToShip);
}
if (CurrentFile.AvailablePages == 0)
{
CurrentFile = null;
}
}
public void WriteBufferToFile(JournalFile journalFile, LowLevelTransaction tx)
{
if (_firstPositionInJournalFile != null)
{
using (var tempTx = new TempPagerTransaction())
{
_lazyTransactionPager.EnsureMapped(tempTx, 0, _lastUsedPage);
var src = _lazyTransactionPager.AcquirePagePointer(tempTx, 0);
journalFile.JournalWriter.WritePages(_firstPositionInJournalFile.Value, src, _lastUsedPage);
}
}
if (tx != null)
{
tx.IsLazyTransaction = false;// so it will notify the flush thread it has work to do
}
_readTransaction?.Dispose();
_firstPositionInJournalFile = null;
_lastUsedPage = 0;
_readTransaction = null;
NumberOfPages = 0;
}
public TimeSpan WriteBufferToFile(JournalFile journalFile, LowLevelTransaction tx)
{
TimeSpan writeToJournalDuration = default;
if (_firstPositionInJournalFile != null)
{
using (var tempTx = new TempPagerTransaction())
{
var numberOfPages = _lastUsed4Kbs / (Constants.Storage.PageSize / (4 * Constants.Size.Kilobyte));
if ((_lastUsed4Kbs % (Constants.Storage.PageSize / (4 * Constants.Size.Kilobyte))) != 0)
{
numberOfPages++;
}
_lazyTransactionPager.EnsureMapped(tempTx, 0, numberOfPages);
var src = _lazyTransactionPager.AcquirePagePointer(tempTx, 0);
var sp = Stopwatch.StartNew();
writeToJournalDuration = journalFile.Write(_firstPositionInJournalFile.Value, src, _lastUsed4Kbs);
if (_log.IsInfoEnabled)
{
_log.Info($"Writing lazy transaction buffer with {_lastUsed4Kbs / 4:#,#0} kb took {sp.Elapsed}");
}
ZeroLazyTransactionBufferIfNeeded(tempTx);
}
}
if (tx != null)
{
tx.IsLazyTransaction = false;// so it will notify the flush thread it has work to do
}
_readTransaction?.Dispose();
_firstPositionInJournalFile = null;
_lastUsed4Kbs = 0;
_readTransaction = null;
NumberOfPages = 0;
return(writeToJournalDuration);
}
public void WriteToJournal(LowLevelTransaction tx, int pageCount)
{
var pages = CompressPages(tx, pageCount, _compressionPager);
if (tx.IsLazyTransaction && _lazyTransactionBuffer == null)
{
_lazyTransactionBuffer = new LazyTransactionBuffer(_env.Options);
}
if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length)
{
_lazyTransactionBuffer?.WriteBufferToFile(CurrentFile, tx);
CurrentFile = NextFile(pages.Length);
}
CurrentFile.Write(tx, pages, _lazyTransactionBuffer, pageCount);
if (CurrentFile.AvailablePages == 0)
{
_lazyTransactionBuffer?.WriteBufferToFile(CurrentFile, tx);
CurrentFile = null;
}
}
public void WriteToJournal(Transaction tx, int pageCount)
{
var pages = CompressPages(tx, pageCount, _compressionPager);
if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length)
{
CurrentFile = NextFile(pages.Length);
}
CurrentFile.Write(tx, pages);
if (CurrentFile.AvailablePages == 0)
{
CurrentFile = null;
}
}
public void ApplyLogsToDataFile(long oldestActiveTransaction, CancellationToken token, Transaction transaction = null)
{
bool locked = false;
if (_flushingSemaphore.IsWriteLockHeld == false)
{
if (_flushingSemaphore.TryEnterWriteLock(Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30)) == false)
{
throw new TimeoutException("Could not acquire the write lock in 30 seconds");
}
locked = true;
}
try
{
if (token.IsCancellationRequested)
{
return;
}
var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite;
var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList();
if (jrnls.Count == 0)
{
return; // nothing to do
}
Debug.Assert(jrnls.First().Number >= _lastSyncedJournal);
var pagesToWrite = new Dictionary <long, JournalFile.PagePosition>();
long lastProcessedJournal = -1;
long previousJournalMaxTransactionId = -1;
long lastFlushedTransactionId = -1;
foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal))
{
var currentJournalMaxTransactionId = -1L;
foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(journalFile.LastTransaction))
{
if (oldestActiveTransaction != 0 &&
pagePosition.Value.TransactionId >= oldestActiveTransaction)
{
// we cannot write this yet, there is a read transaction that might be looking at this
// however, we _aren't_ going to be writing this to the data file, since that would be a
// waste, we would just overwrite that value in the next flush anyway
JournalFile.PagePosition existingPagePosition;
if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) &&
pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber)
{
// remove the page only when it comes from the same journal
// otherwise we can damage the journal's page translation table (PTT)
// because the existing overwrite in a next journal can be filtered out
// so we wouldn't write any page to the data file
pagesToWrite.Remove(pagePosition.Key);
}
continue;
}
if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId)
{
continue;
}
currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId);
if (currentJournalMaxTransactionId < previousJournalMaxTransactionId)
{
throw new InvalidOperationException(
"Journal applicator read beyond the oldest active transaction in the next journal file. " +
"This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId +
", previous journal max ix id: " + previousJournalMaxTransactionId +
", oldest active transaction: " + oldestActiveTransaction);
}
lastProcessedJournal = journalFile.Number;
pagesToWrite[pagePosition.Key] = pagePosition.Value;
lastFlushedTransactionId = currentJournalMaxTransactionId;
}
if (currentJournalMaxTransactionId == -1L)
{
continue;
}
previousJournalMaxTransactionId = currentJournalMaxTransactionId;
}
if (pagesToWrite.Count == 0)
{
return;
}
try
{
ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx);
}
catch (DiskFullException diskFullEx)
{
_waj._env.HandleDataDiskFullException(diskFullEx);
return;
}
var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId);
foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number)))
{
_journalsToDelete.Add(unused.Number, unused);
}
using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite))
{
_lastSyncedJournal = lastProcessedJournal;
_lastSyncedTransactionId = lastFlushedTransactionId;
_lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal);
if (unusedJournals.Count > 0)
{
var lastUnusedJournalNumber = unusedJournals.Last().Number;
_waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber, new List <JournalFile>());
}
if (_waj._files.Count == 0)
{
_waj.CurrentFile = null;
}
FreeScratchPages(unusedJournals, txw);
if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit ||
DateTime.Now - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit)
{
_waj._dataPager.Sync();
UpdateFileHeaderAfterDataFileSync(_lastFlushedJournal, oldestActiveTransaction);
foreach (var toDelete in _journalsToDelete.Values)
{
if (_waj._env.Options.IncrementalBackupEnabled == false)
{
toDelete.DeleteOnClose = true;
}
toDelete.Release();
}
_journalsToDelete.Clear();
_totalWrittenButUnsyncedBytes = 0;
_lastDataFileSyncTime = DateTime.Now;
}
if (txw != null)
{
txw.Commit();
}
}
}
finally
{
if (locked)
{
_flushingSemaphore.ExitWriteLock();
}
}
}
public bool RecoverDatabase(TransactionHeader *txHeader)
{
// note, we don't need to do any concurrency here, happens as a single threaded
// fashion on db startup
var requireHeaderUpdate = false;
var logInfo = _headerAccessor.Get(ptr => ptr->Journal);
if (logInfo.JournalFilesCount == 0)
{
_journalIndex = logInfo.LastSyncedJournal;
return(false);
}
var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1;
if (_env.Options.IncrementalBackupEnabled == false)
{
// we want to check that we cleanup old log files if they aren't needed
// this is more just to be safe than anything else, they shouldn't be there.
var unusedfiles = oldestLogFileStillInUse;
while (true)
{
unusedfiles--;
if (_env.Options.TryDeleteJournal(unusedfiles) == false)
{
break;
}
}
}
var lastSyncedTransactionId = logInfo.LastSyncedTransactionId;
var journalFiles = new List <JournalFile>();
long lastSyncedTxId = -1;
long lastSyncedJournal = logInfo.LastSyncedJournal;
for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++)
{
using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber)))
using (var pager = _env.Options.OpenJournalPager(journalNumber))
{
RecoverCurrentJournalSize(pager);
var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader;
var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader);
journalReader.RecoverAndValidate(_env.Options);
var pagesToWrite = journalReader
.TransactionPageTranslation
.Select(kvp => recoveryPager.Read(kvp.Value.JournalPos))
.OrderBy(x => x.PageNumber)
.ToList();
var lastReadHeaderPtr = journalReader.LastTransactionHeader;
if (lastReadHeaderPtr != null)
{
if (pagesToWrite.Count > 0)
{
ApplyPagesToDataFileFromJournal(pagesToWrite);
}
*txHeader = *lastReadHeaderPtr;
lastSyncedTxId = txHeader->TransactionId;
lastSyncedJournal = journalNumber;
}
if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal)
{
var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize);
var jrnlFile = new JournalFile(jrnlWriter, journalNumber);
jrnlFile.InitFrom(journalReader);
jrnlFile.AddRef(); // creator reference - write ahead log
journalFiles.Add(jrnlFile);
}
if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions
{
requireHeaderUpdate = true;
break;
}
}
}
_files = _files.AppendRange(journalFiles);
Debug.Assert(lastSyncedTxId >= 0);
Debug.Assert(lastSyncedJournal >= 0);
_journalIndex = lastSyncedJournal;
_headerAccessor.Modify(
header =>
{
header->Journal.LastSyncedJournal = lastSyncedJournal;
header->Journal.LastSyncedTransactionId = lastSyncedTxId;
header->Journal.CurrentJournal = lastSyncedJournal;
header->Journal.JournalFilesCount = _files.Count;
header->IncrementalBackup.LastCreatedJournal = _journalIndex;
});
CleanupInvalidJournalFiles(lastSyncedJournal);
CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal);
if (_files.Count > 0)
{
var lastFile = _files.Last();
if (lastFile.AvailablePages >= 2)
{
// it must have at least one page for the next transaction header and one page for data
CurrentFile = lastFile;
}
}
return(requireHeaderUpdate);
}
public void ApplyLogsToDataFile(CancellationToken token, Transaction transaction = null, bool allowToFlushOverwrittenPages = false)
{
if (token.IsCancellationRequested)
{
return;
}
if (Monitor.IsEntered(_flushingLock) && ignoreLockAlreadyTaken == false)
{
throw new InvalidJournalFlushRequest("Applying journals to the data file has been already requested on the same thread");
}
bool lockTaken = false;
try
{
Monitor.TryEnter(_flushingLock, Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30), ref lockTaken);
if (lockTaken == false)
{
throw new TimeoutException("Could not acquire the write lock in 30 seconds");
}
var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite;
var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList();
if (jrnls.Count == 0)
{
return; // nothing to do
}
Debug.Assert(jrnls.First().Number >= _lastSyncedJournal);
var pagesToWrite = new Dictionary <long, PagePosition>();
long lastProcessedJournal = -1;
long previousJournalMaxTransactionId = -1;
long lastFlushedTransactionId = -1;
long oldestActiveTransaction = _waj._env.OldestTransaction;
foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal))
{
var currentJournalMaxTransactionId = -1L;
var iterateLatestAsOf = journalFile.LastTransaction;
if (forcedFlushOfOldPages && forcedIterateJournalsAsOf.HasValue)
{
iterateLatestAsOf = Math.Min(journalFile.LastTransaction, forcedIterateJournalsAsOf.Value);
}
foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(iterateLatestAsOf))
{
if (oldestActiveTransaction != 0 &&
pagePosition.Value.TransactionId >= oldestActiveTransaction)
{
if (pagePosition.Value.IsFreedPageMarker)
{
continue;
}
// we cannot write this yet, there is a read transaction that might be looking at this
// however, we _aren't_ going to be writing this to the data file, since that would be a
// waste, we would just overwrite that value in the next flush anyway
PagePosition existingPagePosition;
if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) &&
pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber)
{
// remove the page only when it comes from the same journal
// otherwise we can damage the journal's page translation table (PTT)
// because the existing overwrite in a next journal can be filtered out
// so we wouldn't write any page to the data file
pagesToWrite.Remove(pagePosition.Key);
}
continue;
}
if (pagePosition.Value.IsFreedPageMarker)
{
// to ensure that we won't overwrite data by a page from the older journal where it wasn't marked as free
// while now it not being considered to be applied to the data file
pagesToWrite.Remove(pagePosition.Key);
continue;
}
if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId)
{
continue;
}
currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId);
if (currentJournalMaxTransactionId < previousJournalMaxTransactionId)
{
throw new InvalidOperationException(
"Journal applicator read beyond the oldest active transaction in the next journal file. " +
"This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId +
", previous journal max ix id: " + previousJournalMaxTransactionId +
", oldest active transaction: " + oldestActiveTransaction);
}
lastProcessedJournal = journalFile.Number;
pagesToWrite[pagePosition.Key] = pagePosition.Value;
lastFlushedTransactionId = currentJournalMaxTransactionId;
}
if (currentJournalMaxTransactionId == -1L)
{
continue;
}
previousJournalMaxTransactionId = currentJournalMaxTransactionId;
}
if (pagesToWrite.Count == 0)
{
if (allowToFlushOverwrittenPages)
{
// we probably filtered out all pages because they have some overwrites and we applied an optimization
// that relays on iterating over pages from end of PTT, however we might want to flush such pages
// in order to allow to free them because the scratch buffer might require them
// so we can flush all pages that we are sure they aren't being read by any transaction
using (ForceFlushingPagesOlderThan(oldestActiveTransaction))
{
ApplyLogsToDataFile(token, transaction, false);
}
}
return;
}
try
{
ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx);
}
catch (DiskFullException diskFullEx)
{
_waj._env.HandleDataDiskFullException(diskFullEx);
return;
}
var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId);
foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number)))
{
_journalsToDelete.Add(unused.Number, unused);
}
using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite).JournalApplicatorTransaction())
{
_lastSyncedJournal = lastProcessedJournal;
_lastSyncedTransactionId = lastFlushedTransactionId;
_lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal);
if (unusedJournals.Count > 0)
{
var lastUnusedJournalNumber = unusedJournals.Last().Number;
_waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber);
}
if (_waj._files.Count == 0)
{
_waj.CurrentFile = null;
}
FreeScratchPages(unusedJournals, txw ?? transaction);
if (txw != null)
{
txw.Commit();
}
}
if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit ||
DateTime.UtcNow - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit)
{
SyncDataFile();
}
}
finally
{
if (lockTaken)
{
Monitor.Exit(_flushingLock);
}
}
}
public void WriteToJournal(Transaction tx, int pageCount)
{
var pages = CompressPages(tx, pageCount, _compressionPager, Shipper.PreviousTransactionCrc);
if (CurrentFile == null || CurrentFile.AvailablePages < pages.Length)
{
CurrentFile = NextFile(pages.Length);
}
CurrentFile.Write(tx, pages);
var transactionHeader = *(TransactionHeader*)pages[0];
var onTransactionCommit = OnTransactionCommit;
if (onTransactionCommit != null)
{
var transactionToShip = new TransactionToShip(transactionHeader)
{
CompressedPages = pages
};
onTransactionCommit(transactionToShip);
}
Shipper.SetPreviousTransaction(transactionHeader.TransactionId, transactionHeader.Crc);
if (CurrentFile.AvailablePages == 0)
CurrentFile = null;
}
public void UpdateFileHeaderAfterDataFileSync(JournalFile file, long oldestActiveTransaction)
{
var txHeaders = stackalloc TransactionHeader[2];
var readTxHeader = &txHeaders[0];
var lastReadTxHeader = txHeaders[1];
var txPos = 0;
while (true)
{
if (file.ReadTransaction(txPos, readTxHeader) == false)
break;
if (readTxHeader->HeaderMarker != Constants.TransactionHeaderMarker)
break;
if (readTxHeader->TransactionId + 1 == oldestActiveTransaction)
break;
lastReadTxHeader = *readTxHeader;
var compressedPages = (readTxHeader->CompressedSize / AbstractPager.PageSize) + (readTxHeader->CompressedSize % AbstractPager.PageSize == 0 ? 0 : 1);
txPos += compressedPages + 1;
}
Debug.Assert(_lastSyncedJournal != -1);
Debug.Assert(_lastSyncedTransactionId != -1);
_waj._headerAccessor.Modify(header =>
{
header->TransactionId = lastReadTxHeader.TransactionId;
header->LastPageNumber = lastReadTxHeader.LastPageNumber;
header->Journal.LastSyncedJournal = _lastSyncedJournal;
header->Journal.LastSyncedTransactionId = _lastSyncedTransactionId;
header->Root = lastReadTxHeader.Root;
header->FreeSpace = lastReadTxHeader.FreeSpace;
header->PreviousTransactionCrc = _waj.Shipper.PreviousTransactionCrc;
_waj._updateLogInfo(header);
});
}
public void ApplyLogsToDataFile(long oldestActiveTransaction, CancellationToken token, Transaction transaction = null, bool allowToFlushOverwrittenPages = false)
{
if (token.IsCancellationRequested)
return;
if (Monitor.IsEntered(_flushingLock) && ignoreLockAlreadyTaken == false)
throw new InvalidJournalFlushRequest("Applying journals to the data file has been already requested on the same thread");
bool lockTaken = false;
try
{
Monitor.TryEnter(_flushingLock, Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30), ref lockTaken);
if (lockTaken == false)
throw new TimeoutException("Could not acquire the write lock in 30 seconds");
var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite;
var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList();
if (jrnls.Count == 0)
return; // nothing to do
Debug.Assert(jrnls.First().Number >= _lastSyncedJournal);
var pagesToWrite = new Dictionary<long, PagePosition>();
long lastProcessedJournal = -1;
long previousJournalMaxTransactionId = -1;
long lastFlushedTransactionId = -1;
foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal))
{
var currentJournalMaxTransactionId = -1L;
var iterateLatestAsOf = journalFile.LastTransaction;
if (forcedFlushOfOldPages && forcedIterateJournalsAsOf.HasValue)
iterateLatestAsOf = Math.Min(journalFile.LastTransaction, forcedIterateJournalsAsOf.Value);
foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(iterateLatestAsOf))
{
if (oldestActiveTransaction != 0 &&
pagePosition.Value.TransactionId >= oldestActiveTransaction)
{
if(pagePosition.Value.IsFreedPageMarker)
continue;
// we cannot write this yet, there is a read transaction that might be looking at this
// however, we _aren't_ going to be writing this to the data file, since that would be a
// waste, we would just overwrite that value in the next flush anyway
PagePosition existingPagePosition;
if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) &&
pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber)
{
// remove the page only when it comes from the same journal
// otherwise we can damage the journal's page translation table (PTT)
// because the existing overwrite in a next journal can be filtered out
// so we wouldn't write any page to the data file
pagesToWrite.Remove(pagePosition.Key);
}
continue;
}
if (pagePosition.Value.IsFreedPageMarker)
{
// to ensure that we won't overwrite data by a page from the older journal where it wasn't marked as free
// while now it not being considered to be applied to the data file
pagesToWrite.Remove(pagePosition.Key);
continue;
}
if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId)
continue;
currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId);
if (currentJournalMaxTransactionId < previousJournalMaxTransactionId)
throw new InvalidOperationException(
"Journal applicator read beyond the oldest active transaction in the next journal file. " +
"This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId +
", previous journal max ix id: " + previousJournalMaxTransactionId +
", oldest active transaction: " + oldestActiveTransaction);
lastProcessedJournal = journalFile.Number;
pagesToWrite[pagePosition.Key] = pagePosition.Value;
lastFlushedTransactionId = currentJournalMaxTransactionId;
}
if (currentJournalMaxTransactionId == -1L)
continue;
previousJournalMaxTransactionId = currentJournalMaxTransactionId;
}
if (pagesToWrite.Count == 0)
{
if (allowToFlushOverwrittenPages)
{
// we probably filtered out all pages because they have some overwrites and we applied an optimization
// that relays on iterating over pages from end of PTT, however we might want to flush such pages
// in order to allow to free them because the scratch buffer might require them
// so we can flush all pages that we are sure they aren't being read by any transaction
using (ForceFlushingPagesOlderThan(oldestActiveTransaction))
{
ApplyLogsToDataFile(oldestActiveTransaction, token, transaction, false);
}
}
return;
}
try
{
ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx);
}
catch (DiskFullException diskFullEx)
{
_waj._env.HandleDataDiskFullException(diskFullEx);
return;
}
var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId);
foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number)))
{
_journalsToDelete.Add(unused.Number, unused);
}
using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite).JournalApplicatorTransaction())
{
_lastSyncedJournal = lastProcessedJournal;
_lastSyncedTransactionId = lastFlushedTransactionId;
_lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal);
if (unusedJournals.Count > 0)
{
var lastUnusedJournalNumber = unusedJournals.Last().Number;
_waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber);
}
if (_waj._files.Count == 0)
_waj.CurrentFile = null;
FreeScratchPages(unusedJournals, txw ?? transaction);
if (txw != null)
txw.Commit();
}
if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit ||
DateTime.UtcNow - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit)
{
SyncDataFile(oldestActiveTransaction);
}
}
finally
{
if(lockTaken)
Monitor.Exit(_flushingLock);
}
}
public void Clear(Transaction tx)
{
if (tx.Flags != TransactionFlags.ReadWrite)
throw new InvalidOperationException("Clearing of write ahead journal should be called only from a write transaction");
foreach (var journalFile in _files)
{
journalFile.Release();
}
_files = ImmutableAppendOnlyList<JournalFile>.Empty;
CurrentFile = null;
}
public bool RecoverDatabase(TransactionHeader* txHeader)
{
// note, we don't need to do any concurrency here, happens as a single threaded
// fashion on db startup
var requireHeaderUpdate = false;
var logInfo = _headerAccessor.Get(ptr => ptr->Journal);
if (logInfo.JournalFilesCount == 0)
{
_journalIndex = logInfo.LastSyncedJournal;
return false;
}
var oldestLogFileStillInUse = logInfo.CurrentJournal - logInfo.JournalFilesCount + 1;
if (_env.Options.IncrementalBackupEnabled == false)
{
// we want to check that we cleanup old log files if they aren't needed
// this is more just to be safe than anything else, they shouldn't be there.
var unusedfiles = oldestLogFileStillInUse;
while (true)
{
unusedfiles--;
if (_env.Options.TryDeleteJournal(unusedfiles) == false)
break;
}
}
var lastSyncedTransactionId = logInfo.LastSyncedTransactionId;
var journalFiles = new List<JournalFile>();
long lastSyncedTxId = -1;
long lastSyncedJournal = logInfo.LastSyncedJournal;
uint lastShippedTxCrc = 0;
for (var journalNumber = oldestLogFileStillInUse; journalNumber <= logInfo.CurrentJournal; journalNumber++)
{
using (var recoveryPager = _env.Options.CreateScratchPager(StorageEnvironmentOptions.JournalRecoveryName(journalNumber)))
using (var pager = _env.Options.OpenJournalPager(journalNumber))
{
RecoverCurrentJournalSize(pager);
var transactionHeader = txHeader->TransactionId == 0 ? null : txHeader;
var journalReader = new JournalReader(pager, recoveryPager, lastSyncedTransactionId, transactionHeader);
journalReader.RecoverAndValidate(_env.Options);
var pagesToWrite = journalReader
.TransactionPageTranslation
.Select(kvp => recoveryPager.Read(kvp.Value.JournalPos))
.OrderBy(x => x.PageNumber)
.ToList();
var lastReadHeaderPtr = journalReader.LastTransactionHeader;
if (lastReadHeaderPtr != null)
{
if (pagesToWrite.Count > 0)
ApplyPagesToDataFileFromJournal(pagesToWrite);
*txHeader = *lastReadHeaderPtr;
lastSyncedTxId = txHeader->TransactionId;
lastShippedTxCrc = txHeader->Crc;
lastSyncedJournal = journalNumber;
}
if (journalReader.RequireHeaderUpdate || journalNumber == logInfo.CurrentJournal)
{
var jrnlWriter = _env.Options.CreateJournalWriter(journalNumber, pager.NumberOfAllocatedPages * AbstractPager.PageSize);
var jrnlFile = new JournalFile(jrnlWriter, journalNumber);
jrnlFile.InitFrom(journalReader);
jrnlFile.AddRef(); // creator reference - write ahead log
journalFiles.Add(jrnlFile);
}
if (journalReader.RequireHeaderUpdate) //this should prevent further loading of transactions
{
requireHeaderUpdate = true;
break;
}
}
}
Shipper.SetPreviousTransaction(lastSyncedTxId, lastShippedTxCrc);
_files = _files.AppendRange(journalFiles);
Debug.Assert(lastSyncedTxId >= 0);
Debug.Assert(lastSyncedJournal >= 0);
_journalIndex = lastSyncedJournal;
_headerAccessor.Modify(
header =>
{
header->Journal.LastSyncedJournal = lastSyncedJournal;
header->Journal.LastSyncedTransactionId = lastSyncedTxId;
header->Journal.CurrentJournal = lastSyncedJournal;
header->Journal.JournalFilesCount = _files.Count;
header->IncrementalBackup.LastCreatedJournal = _journalIndex;
header->PreviousTransactionCrc = lastShippedTxCrc;
});
CleanupInvalidJournalFiles(lastSyncedJournal);
CleanupUnusedJournalFiles(oldestLogFileStillInUse, lastSyncedJournal);
if (_files.Count > 0)
{
var lastFile = _files.Last();
if (lastFile.AvailablePages >= 2)
// it must have at least one page for the next transaction header and one page for data
CurrentFile = lastFile;
}
return requireHeaderUpdate;
}
public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
Action<string> infoNotify = null)
{
infoNotify = infoNotify ?? (s => { });
if (env.Options.IncrementalBackupEnabled == false)
throw new InvalidOperationException("Incremental backup is disabled for this storage");
long numberOfBackedUpPages = 0;
var copier = new DataCopier(AbstractPager.PageSize * 16);
var backupSuccess = true;
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
using (var file = new FileStream(backupPath, FileMode.Create))
using (var package = new ZipArchive(file, ZipArchiveMode.Create))
{
IncrementalBackupInfo backupInfo;
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition;
}
// txw.Commit(); intentionally not committing
}
using (env.NewTransaction(TransactionFlags.Read))
{
var usedJournals = new List<JournalFile>();
try
{
long lastBackedUpPage = -1;
long lastBackedUpFile = -1;
var firstJournalToBackup = backupInfo.LastBackedUpJournal;
if (firstJournalToBackup == -1)
firstJournalToBackup = 0; // first time that we do incremental backup
for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++)
{
var num = journalNum;
var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
}
journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
var startBackupAt = 0L;
var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
if (journalFile.Number == backupInfo.LastBackedUpJournal)
{
startBackupAt = backupInfo.LastBackedUpJournalPage + 1;
pagesToCopy -= startBackupAt;
}
if (startBackupAt >= journalFile.JournalWriter.NumberOfAllocatedPages) // nothing to do here
continue;
var part = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalNum), compression);
Debug.Assert(part != null);
if (journalFile.Number == lastWrittenLogFile)
pagesToCopy -= (journalFile.JournalWriter.NumberOfAllocatedPages - lastWrittenLogPage);
using (var stream = part.Open())
{
copier.ToStream(journalFile, startBackupAt, pagesToCopy, stream);
infoNotify(string.Format("Voron Incr copy journal number {0}", num));
}
lastBackedUpFile = journalFile.Number;
if (journalFile.Number == backupInfo.LastCreatedJournal)
{
lastBackedUpPage = startBackupAt + pagesToCopy - 1;
// we used all of this file, so the next backup should start in the next file
if (lastBackedUpPage == (journalFile.JournalWriter.NumberOfAllocatedPages - 1))
{
lastBackedUpPage = -1;
lastBackedUpFile++;
}
}
numberOfBackedUpPages += pagesToCopy;
}
//Debug.Assert(lastBackedUpPage != -1);
env.HeaderAccessor.Modify(header =>
{
header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile;
header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
});
}
catch (Exception)
{
backupSuccess = false;
throw;
}
finally
{
foreach (var jrnl in usedJournals)
{
if (backupSuccess) // if backup succeeded we can remove journals
{
if (jrnl.Number < lastWrittenLogFile) // prevent deletion of the current journal and journals with a greater number
{
jrnl.DeleteOnClose = true;
}
}
jrnl.Release();
}
}
infoNotify(string.Format("Voron Incr Backup total {0} pages", numberOfBackedUpPages));
return numberOfBackedUpPages;
}
}
}
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
Action<string> infoNotify = null,
Action backupStarted = null)
{
infoNotify = infoNotify ?? (s => { });
var dataPager = env.Options.DataPager;
var copier = new DataCopier(AbstractPager.PageSize * 16);
Transaction txr = null;
try
{
infoNotify("Voron copy headers");
using (var file = new FileStream(backupPath, FileMode.Create))
{
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen:true))
{
long allocatedPages;
ImmutableAppendOnlyList<JournalFile> files; // thread safety copy
var usedJournals = new List<JournalFile>();
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
{
txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view
allocatedPages = dataPager.NumberOfAllocatedPages;
Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options);
// journal files snapshot
files = env.Journal.Files;
JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1; journalNum <= journalInfo.CurrentJournal; journalNum++)
{
var journalFile = files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
}
journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
}
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
}
// txw.Commit(); intentionally not committing
}
if (backupStarted != null)
backupStarted();
// data file backup
var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression);
Debug.Assert(dataPart != null);
if (allocatedPages > 0) //only true if dataPager is still empty at backup start
{
using (var dataStream = dataPart.Open())
{
// now can copy everything else
var firstDataPage = dataPager.Read(0);
copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream);
}
}
try
{
foreach (var journalFile in usedJournals)
{
var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression);
Debug.Assert(journalPart != null);
var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
if (journalFile.Number == lastWrittenLogFile)
pagesToCopy = lastWrittenLogPage + 1;
using (var stream = journalPart.Open())
{
copier.ToStream(journalFile, 0, pagesToCopy, stream);
infoNotify(string.Format("Voron copy journal file {0} ", journalFile));
}
}
}
finally
{
foreach (var journalFile in usedJournals)
{
journalFile.Release();
}
}
}
file.Flush(true); // make sure that we fully flushed to disk
}
}
finally
{
if (txr != null)
txr.Dispose();
}
infoNotify(string.Format("Voron backup db finished"));
}
public void ApplyLogsToDataFile(long oldestActiveTransaction, Transaction transaction = null)
{
bool locked = false;
if (_flushingSemaphore.IsWriteLockHeld == false)
{
if (_flushingSemaphore.TryEnterWriteLock(Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30)) == false)
{
throw new TimeoutException("Could not acquire the write lock in 30 seconds");
}
locked = true;
}
try
{
var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite;
var journalSnapshots = _waj.Files.Select(x => x.GetSnapshot())
.OrderBy(x => x.Number)
.ToList();
if (journalSnapshots.Count == 0)
{
return; // nothing to do
}
Debug.Assert(journalSnapshots.First().Number >= _lastSyncedJournal);
var pagesToWrite = GetPagesFromJournals(oldestActiveTransaction, journalSnapshots);
if (pagesToWrite.Count == 0)
{
return;
}
try
{
ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx);
}
catch (DiskFullException diskFullEx)
{
_waj._env.HandleDataDiskFullException(diskFullEx);
return;
}
var unusedJournals = GetAlreadyHandledJournalFiles(journalSnapshots);
using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite))
{
_lastFlushedJournal = _waj.Files.First(x => x.Number == _lastSyncedJournal);
RemoveUnusedJournalsIfNeeded(unusedJournals);
if (_waj._files.Count == 0)
{
_waj.CurrentFile = null;
}
FreeScratchPages(unusedJournals, txw);
var hasSynced = false;
if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit ||
DateTime.Now - _lastDataFileSyncTime > DelayedDataFileSynchronizationTimeLimit)
{
SyncDataFileWrites(oldestActiveTransaction);
hasSynced = true;
}
if (txw != null)
{
txw.Commit();
}
if (hasSynced)
{
OnApplyLogsToDataFileFinished(_lastSyncedTransactionId);
}
}
}
finally
{
if (locked)
{
_flushingSemaphore.ExitWriteLock();
}
}
}
public void ApplyLogsToDataFile(long oldestActiveTransaction, CancellationToken token, Transaction transaction = null)
{
bool locked = false;
if (_flushingSemaphore.IsWriteLockHeld == false)
{
if (_flushingSemaphore.TryEnterWriteLock(Debugger.IsAttached ? TimeSpan.FromMinutes(30) : TimeSpan.FromSeconds(30)) == false)
throw new TimeoutException("Could not acquire the write lock in 30 seconds");
locked = true;
}
try
{
if (token.IsCancellationRequested)
return;
var alreadyInWriteTx = transaction != null && transaction.Flags == TransactionFlags.ReadWrite;
var jrnls = _waj._files.Select(x => x.GetSnapshot()).OrderBy(x => x.Number).ToList();
if (jrnls.Count == 0)
return; // nothing to do
Debug.Assert(jrnls.First().Number >= _lastSyncedJournal);
var pagesToWrite = new Dictionary<long, JournalFile.PagePosition>();
long lastProcessedJournal = -1;
long previousJournalMaxTransactionId = -1;
long lastFlushedTransactionId = -1;
foreach (var journalFile in jrnls.Where(x => x.Number >= _lastSyncedJournal))
{
var currentJournalMaxTransactionId = -1L;
foreach (var pagePosition in journalFile.PageTranslationTable.IterateLatestAsOf(journalFile.LastTransaction))
{
if (oldestActiveTransaction != 0 &&
pagePosition.Value.TransactionId >= oldestActiveTransaction)
{
// we cannot write this yet, there is a read transaction that might be looking at this
// however, we _aren't_ going to be writing this to the data file, since that would be a
// waste, we would just overwrite that value in the next flush anyway
JournalFile.PagePosition existingPagePosition;
if (pagesToWrite.TryGetValue(pagePosition.Key, out existingPagePosition) &&
pagePosition.Value.JournalNumber == existingPagePosition.JournalNumber)
{
// remove the page only when it comes from the same journal
// otherwise we can damage the journal's page translation table (PTT)
// because the existing overwrite in a next journal can be filtered out
// so we wouldn't write any page to the data file
pagesToWrite.Remove(pagePosition.Key);
}
continue;
}
if (journalFile.Number == _lastSyncedJournal && pagePosition.Value.TransactionId <= _lastSyncedTransactionId)
continue;
currentJournalMaxTransactionId = Math.Max(currentJournalMaxTransactionId, pagePosition.Value.TransactionId);
if (currentJournalMaxTransactionId < previousJournalMaxTransactionId)
throw new InvalidOperationException(
"Journal applicator read beyond the oldest active transaction in the next journal file. " +
"This should never happen. Current journal max tx id: " + currentJournalMaxTransactionId +
", previous journal max ix id: " + previousJournalMaxTransactionId +
", oldest active transaction: " + oldestActiveTransaction);
lastProcessedJournal = journalFile.Number;
pagesToWrite[pagePosition.Key] = pagePosition.Value;
lastFlushedTransactionId = currentJournalMaxTransactionId;
}
if (currentJournalMaxTransactionId == -1L)
continue;
previousJournalMaxTransactionId = currentJournalMaxTransactionId;
}
if (pagesToWrite.Count == 0)
return;
try
{
ApplyPagesToDataFileFromScratch(pagesToWrite, transaction, alreadyInWriteTx);
}
catch (DiskFullException diskFullEx)
{
_waj._env.HandleDataDiskFullException(diskFullEx);
return;
}
var unusedJournals = GetUnusedJournalFiles(jrnls, lastProcessedJournal, lastFlushedTransactionId);
foreach (var unused in unusedJournals.Where(unused => !_journalsToDelete.ContainsKey(unused.Number)))
{
_journalsToDelete.Add(unused.Number, unused);
}
using (var txw = alreadyInWriteTx ? null : _waj._env.NewTransaction(TransactionFlags.ReadWrite))
{
_lastSyncedJournal = lastProcessedJournal;
_lastSyncedTransactionId = lastFlushedTransactionId;
_lastFlushedJournal = _waj._files.First(x => x.Number == _lastSyncedJournal);
if (unusedJournals.Count > 0)
{
var lastUnusedJournalNumber = unusedJournals.Last().Number;
_waj._files = _waj._files.RemoveWhile(x => x.Number <= lastUnusedJournalNumber, new List<JournalFile>());
}
if (_waj._files.Count == 0)
_waj.CurrentFile = null;
FreeScratchPages(unusedJournals, txw);
if (_totalWrittenButUnsyncedBytes > DelayedDataFileSynchronizationBytesLimit ||
DateTime.Now - _lastDataFileSyncTime > _delayedDataFileSynchronizationTimeLimit)
{
_waj._dataPager.Sync();
UpdateFileHeaderAfterDataFileSync(_lastFlushedJournal, oldestActiveTransaction);
foreach (var toDelete in _journalsToDelete.Values)
{
if (_waj._env.Options.IncrementalBackupEnabled == false)
toDelete.DeleteOnClose = true;
toDelete.Release();
}
_journalsToDelete.Clear();
_totalWrittenButUnsyncedBytes = 0;
_lastDataFileSyncTime = DateTime.Now;
}
if (txw != null)
txw.Commit();
}
}
finally
{
if (locked)
_flushingSemaphore.ExitWriteLock();
}
}
internal static JournalFile GetJournalFile(StorageEnvironment env, long journalNum, IncrementalBackupInfo backupInfo)
{
var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
if (journalFile != null)
{
journalFile.AddRef();
return journalFile;
}
try
{
using (var pager = env.Options.OpenJournalPager(journalNum))
{
long journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
journalFile.AddRef();
return journalFile;
}
}
catch (Exception e)
{
if (backupInfo.LastBackedUpJournal == -1 && journalNum == 0 && e.Message.StartsWith("No such journal"))
{
throw new InvalidOperationException("The first incremental backup creation failed because the first journal file " +
StorageEnvironmentOptions.JournalName(journalNum) + " was not found. " +
"Did you turn on the incremental backup feature after initializing the storage? " +
"In order to create backups incrementally the storage must be created with IncrementalBackupEnabled option set to 'true'.", e);
}
throw;
}
}
