/// <summary>
/// Select record format for the given store directory.
/// <para>
/// <b>Note:</b> package private only for testing.
///
/// </para>
/// </summary>
/// <param name="databaseLayout"> directory with the store </param>
/// <param name="fs"> file system used to access store files </param>
/// <param name="pageCache"> page cache to read store files </param>
/// <returns> record format of the given store or <code>null</code> if <seealso cref="DatabaseLayout.metadataStore()"/> file not
/// found or can't be read </returns>
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Nullable static RecordFormats selectForStore(org.neo4j.io.layout.DatabaseLayout databaseLayout, org.neo4j.io.fs.FileSystemAbstraction fs, org.neo4j.io.pagecache.PageCache pageCache, org.neo4j.logging.LogProvider logProvider)
internal static RecordFormats SelectForStore(DatabaseLayout databaseLayout, FileSystemAbstraction fs, PageCache pageCache, LogProvider logProvider)
{
File neoStoreFile = databaseLayout.MetadataStore();
if (fs.FileExists(neoStoreFile))
{
try
{
long value = MetaDataStore.getRecord(pageCache, neoStoreFile, STORE_VERSION);
if (value != MetaDataRecordFormat.FIELD_NOT_PRESENT)
{
string storeVersion = MetaDataStore.versionLongToString(value);
foreach (RecordFormats format in AllFormats())
{
if (format.StoreVersion().Equals(storeVersion))
{
Info(logProvider, "Selected " + format + " record format from store " + databaseLayout.DatabaseDirectory());
return(format);
}
}
}
}
catch (IOException e)
{
Info(logProvider, "Unable to read store format: " + e.Message);
}
}
return(null);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: org.neo4j.kernel.impl.transaction.log.LogPosition extractTransactionLogPosition(java.io.File neoStore, org.neo4j.io.layout.DatabaseLayout sourceDirectoryStructure, long lastTxId) throws java.io.IOException
internal virtual LogPosition ExtractTransactionLogPosition(File neoStore, DatabaseLayout sourceDirectoryStructure, long lastTxId)
{
long lastClosedTxLogVersion = MetaDataStore.getRecord(_pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION);
long lastClosedTxLogByteOffset = MetaDataStore.getRecord(_pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET);
if (lastClosedTxLogVersion != MetaDataRecordFormat.FIELD_NOT_PRESENT && lastClosedTxLogByteOffset != MetaDataRecordFormat.FIELD_NOT_PRESENT)
{
return(new LogPosition(lastClosedTxLogVersion, lastClosedTxLogByteOffset));
}
// The legacy store we're migrating doesn't have this record in neostore so try to extract it from tx log
if (lastTxId == Org.Neo4j.Kernel.impl.transaction.log.TransactionIdStore_Fields.BASE_TX_ID)
{
return(new LogPosition(BASE_TX_LOG_VERSION, BASE_TX_LOG_BYTE_OFFSET));
}
LogFiles logFiles = LogFilesBuilder.activeFilesBuilder(sourceDirectoryStructure, _fileSystem, _pageCache).withConfig(_config).build();
long logVersion = logFiles.HighestLogVersion;
if (logVersion == -1)
{
return(new LogPosition(BASE_TX_LOG_VERSION, BASE_TX_LOG_BYTE_OFFSET));
}
long offset = _fileSystem.getFileSize(logFiles.HighestLogFile);
return(new LogPosition(logVersion, offset));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public java.util.Optional<String> getVersion(java.io.File neostoreFile) throws java.io.IOException
public virtual Optional <string> GetVersion(File neostoreFile)
{
long record = MetaDataStore.getRecord(_pageCache, neostoreFile, STORE_VERSION);
if (record == MetaDataRecordFormat.FIELD_NOT_PRESENT)
{
return(null);
}
return(MetaDataStore.versionLongToString(record));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private static org.neo4j.causalclustering.identity.StoreId doReadStoreId(java.io.File databaseDirectory, org.neo4j.io.pagecache.PageCache pageCache) throws java.io.IOException
private static StoreId DoReadStoreId(File databaseDirectory, PageCache pageCache)
{
File metadataStore = DatabaseLayout.of(databaseDirectory).metadataStore();
long creationTime = MetaDataStore.getRecord(pageCache, metadataStore, TIME);
long randomNumber = MetaDataStore.getRecord(pageCache, metadataStore, RANDOM_NUMBER);
long upgradeTime = MetaDataStore.getRecord(pageCache, metadataStore, UPGRADE_TIME);
long upgradeId = MetaDataStore.getRecord(pageCache, metadataStore, UPGRADE_TRANSACTION_ID);
return(new StoreId(creationTime, randomNumber, upgradeTime, upgradeId));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: org.neo4j.kernel.impl.store.TransactionId extractTransactionIdInformation(java.io.File neoStore, long lastTransactionId) throws java.io.IOException
internal virtual TransactionId ExtractTransactionIdInformation(File neoStore, long lastTransactionId)
{
long checksum = MetaDataStore.getRecord(_pageCache, neoStore, MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM);
long commitTimestamp = MetaDataStore.getRecord(_pageCache, neoStore, MetaDataStore.Position.LAST_TRANSACTION_COMMIT_TIMESTAMP);
if (checksum != FIELD_NOT_PRESENT && commitTimestamp != FIELD_NOT_PRESENT)
{
return(new TransactionId(lastTransactionId, checksum, commitTimestamp));
}
return(SpecificTransactionInformationSupplier(lastTransactionId));
}
private static long LastAppliedTx(DatabaseLayout databaseLayout)
{
try
{
using (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(), JobScheduler scheduler = createInitialisedScheduler(), PageCache pageCache = createPageCache(fileSystem, scheduler))
{
return(MetaDataStore.getRecord(pageCache, databaseLayout.MetadataStore(), MetaDataStore.Position.LAST_TRANSACTION_ID));
}
}
catch (Exception e)
{
throw new Exception(e);
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public void migrate(org.neo4j.io.layout.DatabaseLayout directoryLayout, org.neo4j.io.layout.DatabaseLayout migrationLayout, org.neo4j.kernel.impl.util.monitoring.ProgressReporter progressReporter, String versionToMigrateFrom, String versionToMigrateTo) throws java.io.IOException
public override void Migrate(DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressReporter, string versionToMigrateFrom, string versionToMigrateTo)
{
// Extract information about the last transaction from legacy neostore
File neoStore = directoryLayout.MetadataStore();
long lastTxId = MetaDataStore.getRecord(_pageCache, neoStore, MetaDataStore.Position.LAST_TRANSACTION_ID);
TransactionId lastTxInfo = ExtractTransactionIdInformation(neoStore, lastTxId);
LogPosition lastTxLogPosition = ExtractTransactionLogPosition(neoStore, directoryLayout, lastTxId);
// Write the tx checksum to file in migrationStructure, because we need it later when moving files into storeDir
WriteLastTxInformation(migrationLayout, lastTxInfo);
WriteLastTxLogPosition(migrationLayout, lastTxLogPosition);
if (versionToMigrateFrom.Equals("vE.H.0"))
{
// NOTE for 3.0 here is a special case for vE.H.0 "from" record format.
// Legend has it that 3.0.5 enterprise changed store format without changing store version.
// This was done to cheat the migrator to avoid doing store migration since the
// format itself was backwards compatible. Immediately a problem was detected:
// if a user uses 3.0.5 for a while and then goes back to a previous 3.0.x patch release
// the db wouldn't recognize it was an incompatible downgrade and start up normally,
// but read records with scrambled values and pointers, sort of.
//
// This condition has two functions:
// 1. preventing actual store migration between vE.H.0 --> vE.H.0b
// 2. making vE.H.0b used in any migration where either vE.H.0 or vE.H.0b is the existing format,
// this because vE.H.0b is a superset of vE.H.0 and sometimes (for 3.0.5) vE.H.0
// actually means vE.H.0b (in later version).
//
// In later versions of neo4j there are better mechanics in place so that a non-migration like this
// can be performed w/o special casing. To not require backporting that functionality
// this condition is here and should be removed in 3.1.
versionToMigrateFrom = "vE.H.0b";
}
RecordFormats oldFormat = selectForVersion(versionToMigrateFrom);
RecordFormats newFormat = selectForVersion(versionToMigrateTo);
if (FormatFamily.isHigherFamilyFormat(newFormat, oldFormat) || (FormatFamily.isSameFamily(oldFormat, newFormat) && IsDifferentCapabilities(oldFormat, newFormat)))
{
// TODO if this store has relationship indexes then warn user about that they will be incorrect
// after migration, because now we're rewriting the relationship ids.
// Some form of migration is required (a fallback/catch-all option)
MigrateWithBatchImporter(directoryLayout, migrationLayout, lastTxId, lastTxInfo.Checksum(), lastTxLogPosition.LogVersion, lastTxLogPosition.ByteOffset, progressReporter, oldFormat, newFormat);
}
// update necessary neostore records
LogPosition logPosition = ReadLastTxLogPosition(migrationLayout);
UpdateOrAddNeoStoreFieldsAsPartOfMigration(migrationLayout, directoryLayout, versionToMigrateTo, logPosition);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public void migrate(org.neo4j.io.layout.DatabaseLayout directoryLayout, org.neo4j.io.layout.DatabaseLayout migrationLayout, org.neo4j.kernel.impl.util.monitoring.ProgressReporter progressMonitor, String versionToMigrateFrom, String versionToMigrateTo) throws java.io.IOException
public override void Migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressMonitor, string versionToMigrateFrom, string versionToMigrateTo )
{
if ( CountStoreRebuildRequired( versionToMigrateFrom ) )
{
// create counters from scratch
fileOperation( DELETE, _fileSystem, migrationLayout, migrationLayout, _countsStoreFiles, true, null );
File neoStore = directoryLayout.MetadataStore();
long lastTxId = MetaDataStore.getRecord( _pageCache, neoStore, MetaDataStore.Position.LAST_TRANSACTION_ID );
try
{
RebuildCountsFromScratch( directoryLayout, migrationLayout, lastTxId, progressMonitor, versionToMigrateTo, _pageCache, NullLogProvider.Instance );
}
catch ( StoreFailureException )
{
//This means that we did not perform a full migration, as the formats had the same capabilities. Thus
// we should use the store directory for information when rebuilding the count store. Note that we
// still put the new count store in the migration directory.
RebuildCountsFromScratch( directoryLayout, migrationLayout, lastTxId, progressMonitor, versionToMigrateFrom, _pageCache, NullLogProvider.Instance );
}
_migrated = true;
}
}
internal virtual BranchedStoreInfo ParseBranchedStore(File branchedDatabase)
{
try
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.io.File neoStoreFile = org.neo4j.io.layout.DatabaseLayout.of(branchedDatabase).metadataStore();
File neoStoreFile = DatabaseLayout.of(branchedDatabase).metadataStore();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long txId = org.neo4j.kernel.impl.store.MetaDataStore.getRecord(pageCache, neoStoreFile, org.neo4j.kernel.impl.store.MetaDataStore.Position.LAST_TRANSACTION_ID);
long txId = MetaDataStore.getRecord(PageCache, neoStoreFile, MetaDataStore.Position.LAST_TRANSACTION_ID);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long timestamp = long.Parse(branchedDatabase.getName());
long timestamp = long.Parse(branchedDatabase.Name);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final long branchedStoreSize = org.neo4j.io.fs.FileUtils.size(fileSystem, branchedDatabase);
long branchedStoreSize = FileUtils.size(FileSystem, branchedDatabase);
return(new BranchedStoreInfo(branchedDatabase.Name, txId, timestamp, branchedStoreSize));
}
catch (IOException e)
{
throw new System.InvalidOperationException("Cannot read branched neostore", e);
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void updateOrAddNeoStoreFieldsAsPartOfMigration(org.neo4j.io.layout.DatabaseLayout migrationStructure, org.neo4j.io.layout.DatabaseLayout sourceDirectoryStructure, String versionToMigrateTo, org.neo4j.kernel.impl.transaction.log.LogPosition lastClosedTxLogPosition) throws java.io.IOException
private void UpdateOrAddNeoStoreFieldsAsPartOfMigration(DatabaseLayout migrationStructure, DatabaseLayout sourceDirectoryStructure, string versionToMigrateTo, LogPosition lastClosedTxLogPosition)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.io.File storeDirNeoStore = sourceDirectoryStructure.metadataStore();
File storeDirNeoStore = sourceDirectoryStructure.MetadataStore();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.io.File migrationDirNeoStore = migrationStructure.metadataStore();
File migrationDirNeoStore = migrationStructure.MetadataStore();
fileOperation(COPY, _fileSystem, sourceDirectoryStructure, migrationStructure, Iterables.iterable(DatabaseFile.METADATA_STORE), true, ExistingTargetStrategy.SKIP);
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.UPGRADE_TRANSACTION_ID, MetaDataStore.getRecord(_pageCache, storeDirNeoStore, MetaDataStore.Position.LAST_TRANSACTION_ID));
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.UPGRADE_TIME, DateTimeHelper.CurrentUnixTimeMillis());
// Store the checksum of the transaction id the upgrade is at right now. Store it both as
// LAST_TRANSACTION_CHECKSUM and UPGRADE_TRANSACTION_CHECKSUM. Initially the last transaction and the
// upgrade transaction will be the same, but imagine this scenario:
// - legacy store is migrated on instance A at transaction T
// - upgraded store is copied, via backup or HA or whatever to instance B
// - instance A performs a transaction
// - instance B would like to communicate with A where B's last transaction checksum
// is verified on A. A, at this point not having logs from pre-migration era, will need to
// know the checksum of transaction T to accommodate for this request from B. A will be able
// to look up checksums for transactions succeeding T by looking at its transaction logs,
// but T needs to be stored in neostore to be accessible. Obviously this scenario is only
// problematic as long as we don't migrate and translate old logs.
TransactionId lastTxInfo = ReadLastTxInformation(migrationStructure);
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM, lastTxInfo.Checksum());
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.UPGRADE_TRANSACTION_CHECKSUM, lastTxInfo.Checksum());
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.LAST_TRANSACTION_COMMIT_TIMESTAMP, lastTxInfo.CommitTimestamp());
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.UPGRADE_TRANSACTION_COMMIT_TIMESTAMP, lastTxInfo.CommitTimestamp());
// add LAST_CLOSED_TRANSACTION_LOG_VERSION and LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET to the migrated
// NeoStore
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION, lastClosedTxLogPosition.LogVersion);
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, lastClosedTxLogPosition.ByteOffset);
// Upgrade version in NeoStore
MetaDataStore.setRecord(_pageCache, migrationDirNeoStore, MetaDataStore.Position.STORE_VERSION, MetaDataStore.versionStringToLong(versionToMigrateTo));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private static long lastTxChecksumOf(org.neo4j.io.layout.DatabaseLayout databaseLayout, org.neo4j.io.pagecache.PageCache pageCache) throws java.io.IOException
private static long LastTxChecksumOf(DatabaseLayout databaseLayout, PageCache pageCache)
{
File neoStore = databaseLayout.MetadataStore();
return(MetaDataStore.getRecord(pageCache, neoStore, MetaDataStore.Position.LAST_TRANSACTION_CHECKSUM));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private static long getLastCommittedTx(org.neo4j.io.layout.DatabaseLayout databaseLayout, org.neo4j.io.pagecache.PageCache pageCache) throws java.io.IOException
private static long GetLastCommittedTx(DatabaseLayout databaseLayout, PageCache pageCache)
{
File neoStore = databaseLayout.MetadataStore();
return(MetaDataStore.getRecord(pageCache, neoStore, MetaDataStore.Position.LAST_TRANSACTION_ID));
}