/// <summary>
/// Rotates the current log file, continuing into next (version) log file.
/// This method must be recovery safe, which means a crash at any point should be recoverable.
/// Concurrent readers must also be able to parry for concurrent rotation.
/// Concurrent writes will not be an issue since rotation and writing contends on the same monitor.
///
/// Steps during rotation are:
/// <ol>
/// <li>1: Increment log version, <seealso cref="LogVersionRepository.incrementAndGetVersion()"/> (also flushes the store)</li>
/// <li>2: Flush current log</li>
/// <li>3: Create new log file</li>
/// <li>4: Write header</li>
/// </ol>
///
/// Recovery: what happens if crash between:
/// <ol>
/// <li>1-2: New log version has been set, starting the writer will create the new log file idempotently.
/// At this point there may have been half-written transactions in the previous log version,
/// although they haven't been considered committed and so they will be truncated from log during recovery</li>
/// <li>2-3: New log version has been set, starting the writer will create the new log file idempotently.
/// At this point there may be complete transactions in the previous log version which may not have been
/// acknowledged to be committed back to the user, but will be considered committed anyway.</li>
/// <li>3-4: New log version has been set, starting the writer will see that the new file exists and
/// will be forgiving when trying to read the header of it, so that if it isn't complete a fresh
/// header will be set.</li>
/// </ol>
///
/// Reading: what happens when rotation is between:
/// <ol>
/// <li>1-2: Reader bridge will see that there's a new version (when asking <seealso cref="LogVersionRepository"/>
/// and try to open it. The log file doesn't exist yet though. The bridge can parry for this by catching
/// <seealso cref="FileNotFoundException"/> and tell the reader that the stream has ended</li>
/// <li>2-3: Same as (1-2)</li>
/// <li>3-4: Here the new log file exists, but the header may not be fully written yet.
/// the reader will fail when trying to read the header since it's reading it strictly and bridge
/// catches that exception, treating it the same as if the file didn't exist.</li>
/// </ol>
/// </summary>
/// <param name="currentLog"> current <seealso cref="LogVersionedStoreChannel channel"/> to flush and close. </param>
/// <returns> the channel of the newly opened/created log file. </returns>
/// <exception cref="IOException"> if an error regarding closing or opening log files occur. </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel rotate(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel currentLog) throws java.io.IOException
private PhysicalLogVersionedStoreChannel Rotate(LogVersionedStoreChannel currentLog)
{
/*
* The store is now flushed. If we fail now the recovery code will open the
* current log file and replay everything. That's unnecessary but totally ok.
*/
long newLogVersion = _logVersionRepository.incrementAndGetVersion();
/*
* Rotation can happen at any point, although not concurrently with an append,
* although an append may have (most likely actually) left at least some bytes left
* in the buffer for future flushing. Flushing that buffer now makes the last appended
* transaction complete in the log we're rotating away. Awesome.
*/
_writer.prepareForFlush().flush();
/*
* The log version is now in the store, flushed and persistent. If we crash
* now, on recovery we'll attempt to open the version we're about to create
* (but haven't yet), discover it's not there. That will lead to creating
* the file, setting the header and continuing.
* We using committing transaction id as a source of last transaction id here since
* we can have transactions that are not yet published as committed but were already stored
* into transaction log that was just rotated.
*/
PhysicalLogVersionedStoreChannel newLog = _logFiles.createLogChannelForVersion(newLogVersion, OpenMode.READ_WRITE, _context.committingTransactionId);
currentLog.close();
return(newLog);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public LogVersionedStoreChannel next(LogVersionedStoreChannel channel) throws java.io.IOException
public override LogVersionedStoreChannel Next(LogVersionedStoreChannel channel)
{
PhysicalLogVersionedStoreChannel nextChannel;
try
{
nextChannel = _logFiles.openForVersion(channel.Version + 1);
}
catch (Exception e) when(e is FileNotFoundException || e is IncompleteLogHeaderException)
{
// See PhysicalLogFile#rotate() for description as to why these exceptions are OK
return(channel);
}
channel.close();
return(nextChannel);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private int transactionCount() throws java.io.IOException
private int TransactionCount()
{
return(AggregateLogData(version =>
{
int counter = 0;
LogVersionBridge bridge = channel => channel;
LogVersionedStoreChannel versionedStoreChannel = _files.openForVersion(version);
using (ReadableLogChannel channel = new ReadAheadLogChannel(versionedStoreChannel, bridge, 1000))
{
using (PhysicalTransactionCursor <ReadableLogChannel> physicalTransactionCursor = new PhysicalTransactionCursor <ReadableLogChannel>(channel, new VersionAwareLogEntryReader <>()))
{
while (physicalTransactionCursor.Next())
{
counter++;
}
}
}
return counter;
}));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void verifyTransactionsInLog(org.neo4j.kernel.impl.transaction.log.files.LogFiles logFiles, long fromTxId, long endTxId) throws java.io.IOException
private void VerifyTransactionsInLog(LogFiles logFiles, long fromTxId, long endTxId)
{
long expectedTxId = fromTxId;
LogVersionedStoreChannel versionedStoreChannel = logFiles.OpenForVersion(0);
using (ReadableLogChannel channel = new ReadAheadLogChannel(versionedStoreChannel, Org.Neo4j.Kernel.impl.transaction.log.LogVersionBridge_Fields.NoMoreChannels, 1024))
{
using (PhysicalTransactionCursor <ReadableLogChannel> txCursor = new PhysicalTransactionCursor <ReadableLogChannel>(channel, new VersionAwareLogEntryReader <>()))
{
while (txCursor.Next())
{
CommittedTransactionRepresentation tx = txCursor.Get();
long txId = tx.CommitEntry.TxId;
assertThat(expectedTxId, lessThanOrEqualTo(endTxId));
assertEquals(expectedTxId, txId);
expectedTxId++;
}
}
}
}
/// <summary>
/// Extracts txId from first commit entry, when starting reading at the given {@code position}.
/// If no commit entry found in the version, the reader will continue into next version(s) up till
/// {@code maxLogVersion} until finding one.
/// </summary>
/// <param name="initialPosition"> <seealso cref="LogPosition"/> to start scan from. </param>
/// <param name="maxLogVersion"> max log version to scan. </param>
/// <returns> value object that contains first transaction id of closes commit entry to {@code initialPosition},
/// or <seealso cref="LogTailInformation.NO_TRANSACTION_ID"/> if not found. And failure flag that will be set to true if
/// there was some exception during transaction log processing. </returns>
/// <exception cref="IOException"> on channel close I/O error. </exception>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: protected ExtractedTransactionRecord extractFirstTxIdAfterPosition(org.neo4j.kernel.impl.transaction.log.LogPosition initialPosition, long maxLogVersion) throws java.io.IOException
protected internal virtual ExtractedTransactionRecord ExtractFirstTxIdAfterPosition(LogPosition initialPosition, long maxLogVersion)
{
LogPosition currentPosition = initialPosition;
while (currentPosition.LogVersion <= maxLogVersion)
{
LogVersionedStoreChannel storeChannel = TryOpenStoreChannel(currentPosition);
if (storeChannel != null)
{
try
{
storeChannel.Position(currentPosition.ByteOffset);
using (ReadAheadLogChannel logChannel = new ReadAheadLogChannel(storeChannel), LogEntryCursor cursor = new LogEntryCursor(_logEntryReader, logChannel))
{
while (cursor.Next())
{
LogEntry entry = cursor.Get();
if (entry is LogEntryCommit)
{
return(new ExtractedTransactionRecord((( LogEntryCommit )entry).TxId));
}
}
}
}
catch (Exception t)
{
_monitor.corruptedLogFile(currentPosition.LogVersion, t);
return(new ExtractedTransactionRecord(true));
}
finally
{
storeChannel.close();
}
}
currentPosition = LogPosition.start(currentPosition.LogVersion + 1);
}
return(new ExtractedTransactionRecord());
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public java.util.List<org.neo4j.kernel.impl.transaction.log.entry.CheckPoint> find(long version) throws java.io.IOException
public virtual IList <CheckPoint> Find(long version)
{
IList <CheckPoint> checkPoints = new List <CheckPoint>();
for ( ; version >= INITIAL_LOG_VERSION && LogFiles.versionExists(version); version--)
{
LogVersionedStoreChannel channel = LogFiles.openForVersion(version);
ReadableClosablePositionAwareChannel recoveredDataChannel = new ReadAheadLogChannel(channel);
using (LogEntryCursor cursor = new LogEntryCursor(LogEntryReader, recoveredDataChannel))
{
while (cursor.Next())
{
LogEntry entry = cursor.Get();
if (entry is CheckPoint)
{
checkPoints.Add(entry.As());
}
}
}
}
return(checkPoints);
}
public PositionAwarePhysicalFlushableChannel(LogVersionedStoreChannel logVersionedStoreChannel, int bufferSize)
{
this._logVersionedStoreChannel = logVersionedStoreChannel;
this._channel = new PhysicalFlushableChannel(logVersionedStoreChannel, bufferSize);
}
public PositionAwarePhysicalFlushableChannel(LogVersionedStoreChannel logVersionedStoreChannel)
{
this._logVersionedStoreChannel = logVersionedStoreChannel;
this._channel = new PhysicalFlushableChannel(logVersionedStoreChannel);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private LogTailInformation findLogTail() throws java.io.IOException
private LogTailInformation FindLogTail()
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long highestLogVersion = logFiles.getHighestLogVersion();
long highestLogVersion = _logFiles.HighestLogVersion;
long version = highestLogVersion;
long versionToSearchForCommits = highestLogVersion;
LogEntryStart latestStartEntry = null;
long oldestStartEntryTransaction = -1;
long oldestVersionFound = -1;
LogEntryVersion latestLogEntryVersion = null;
bool startRecordAfterCheckpoint = false;
bool corruptedTransactionLogs = false;
while (version >= _logFiles.LowestLogVersion && version >= INITIAL_LOG_VERSION)
{
oldestVersionFound = version;
CheckPoint latestCheckPoint = null;
try
{
using (LogVersionedStoreChannel channel = _logFiles.openForVersion(version), ReadAheadLogChannel readAheadLogChannel = new ReadAheadLogChannel(channel), LogEntryCursor cursor = new LogEntryCursor(_logEntryReader, readAheadLogChannel))
{
LogEntry entry;
long maxEntryReadPosition = 0;
while (cursor.Next())
{
entry = cursor.Get();
// Collect data about latest checkpoint
if (entry is CheckPoint)
{
latestCheckPoint = entry.As();
}
else if (entry is LogEntryCommit)
{
if (oldestStartEntryTransaction == NoTransactionId)
{
oldestStartEntryTransaction = (( LogEntryCommit )entry).TxId;
}
}
else if (entry is LogEntryStart)
{
LogEntryStart startEntry = entry.As();
if (version == versionToSearchForCommits)
{
latestStartEntry = startEntry;
}
startRecordAfterCheckpoint = true;
}
// Collect data about latest entry version, only in first log file
if (version == versionToSearchForCommits || latestLogEntryVersion == null)
{
latestLogEntryVersion = entry.Version;
}
maxEntryReadPosition = readAheadLogChannel.Position();
}
if (HasUnreadableBytes(channel, maxEntryReadPosition))
{
corruptedTransactionLogs = true;
}
}
}
catch (Exception e) when(e is Exception || e is ClosedByInterruptException)
{
// These should not be parsing errors
throw e;
}
catch (Exception t)
{
_monitor.corruptedLogFile(version, t);
if (_failOnCorruptedLogFiles)
{
throwUnableToCleanRecover(t);
}
corruptedTransactionLogs = true;
}
if (latestCheckPoint != null)
{
return(CheckpointTailInformation(highestLogVersion, latestStartEntry, oldestVersionFound, latestLogEntryVersion, latestCheckPoint, corruptedTransactionLogs));
}
version--;
// if we have found no commits in the latest log, keep searching in the next one
if (latestStartEntry == null)
{
versionToSearchForCommits--;
}
}
return(new LogTailInformation(corruptedTransactionLogs || startRecordAfterCheckpoint, oldestStartEntryTransaction, oldestVersionFound, highestLogVersion, latestLogEntryVersion));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private boolean hasUnreadableBytes(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel channel, long maxEntryReadEndPosition) throws java.io.IOException
private bool HasUnreadableBytes(LogVersionedStoreChannel channel, long maxEntryReadEndPosition)
{
return(channel.position() > maxEntryReadEndPosition);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void databasePanicShouldHandleOutOfMemoryErrors() throws java.io.IOException, InterruptedException
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void DatabasePanicShouldHandleOutOfMemoryErrors()
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.concurrent.CountDownLatch panicLatch = new java.util.concurrent.CountDownLatch(1);
System.Threading.CountdownEvent panicLatch = new System.Threading.CountdownEvent(1);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.concurrent.CountDownLatch adversaryLatch = new java.util.concurrent.CountDownLatch(1);
System.Threading.CountdownEvent adversaryLatch = new System.Threading.CountdownEvent(1);
OutOfMemoryAwareFileSystem fs = new OutOfMemoryAwareFileSystem();
_life.add(new FileSystemLifecycleAdapter(fs));
DatabaseHealth slowPanicDatabaseHealth = new SlowPanickingDatabaseHealth(panicLatch, adversaryLatch);
LogFiles logFiles = LogFilesBuilder.builder(_testDirectory.databaseLayout(), fs).withLogVersionRepository(_logVersionRepository).withTransactionIdStore(_transactionIdStore).build();
_life.add(logFiles);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final BatchingTransactionAppender appender = life.add(new BatchingTransactionAppender(logFiles, logRotation, transactionMetadataCache, transactionIdStore, explicitIndexTransactionOrdering, slowPanicDatabaseHealth));
BatchingTransactionAppender appender = _life.add(new BatchingTransactionAppender(logFiles, _logRotation, _transactionMetadataCache, _transactionIdStore, _explicitIndexTransactionOrdering, slowPanicDatabaseHealth));
_life.start();
// Commit initial transaction
appender.Append(Tx(), Org.Neo4j.Kernel.impl.transaction.tracing.LogAppendEvent_Fields.Null);
// Try to commit one transaction, will fail during flush with OOM, but not actually panic
fs.ShouldOOM = true;
Future <long> failingTransaction = _executor.submit(() => appender.Append(Tx(), Org.Neo4j.Kernel.impl.transaction.tracing.LogAppendEvent_Fields.Null));
panicLatch.await();
// Try to commit one additional transaction, should fail since database has already panicked
fs.ShouldOOM = false;
try
{
appender.Append(Tx(), new LogAppendEvent_EmptyAnonymousInnerClass2(this, adversaryLatch));
fail("Should have failed since database should have panicked");
}
catch (IOException e)
{
assertTrue(e.Message.contains("The database has encountered a critical error"));
}
// Check that we actually got an OutOfMemoryError
try
{
failingTransaction.get();
fail("Should have failed with OutOfMemoryError error");
}
catch (ExecutionException e)
{
assertTrue(e.InnerException is System.OutOfMemoryException);
}
// Check number of transactions, should only have one
LogEntryReader <ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader <ReadableClosablePositionAwareChannel>();
assertEquals(logFiles.LowestLogVersion, logFiles.HighestLogVersion);
long version = logFiles.HighestLogVersion;
using (LogVersionedStoreChannel channel = logFiles.OpenForVersion(version), ReadAheadLogChannel readAheadLogChannel = new ReadAheadLogChannel(channel), LogEntryCursor cursor = new LogEntryCursor(logEntryReader, readAheadLogChannel))
{
LogEntry entry;
long numberOfTransactions = 0;
while (cursor.Next())
{
entry = cursor.Get();
if (entry is LogEntryCommit)
{
numberOfTransactions++;
}
}
assertEquals(1, numberOfTransactions);
}
}
