private StoreFactory StoreFactory(Config config, params OpenOption[] openOptions)
{
LogProvider logProvider = NullLogProvider.Instance;
DatabaseLayout databaseLayout = _testDirectory.databaseLayout();
RecordFormats recordFormats = selectForStoreOrConfig(config, databaseLayout, _fsRule, _pageCache, logProvider);
return(new StoreFactory(databaseLayout, config, _idGeneratorFactory, _pageCache, _fsRule.get(), recordFormats, logProvider, EmptyVersionContextSupplier.EMPTY, openOptions));
}
private StoreFactory GetStoreFactory(DatabaseLayout directoryStructure, string versionToMigrateFrom)
{
NullLogProvider logProvider = NullLogProvider.Instance;
RecordFormats recordFormats = selectForVersion(versionToMigrateFrom);
IdGeneratorFactory idGeneratorFactory = new ReadOnlyIdGeneratorFactory(_fileSystem);
return(new StoreFactory(directoryStructure, _config, idGeneratorFactory, _pageCache, _fileSystem, recordFormats, logProvider, EmptyVersionContextSupplier.EMPTY));
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldNotWantIfCountWayLowerThanCapacity()
public virtual void ShouldNotWantIfCountWayLowerThanCapacity()
{
// given
RecordFormats formats = Standard.LATEST_RECORD_FORMATS;
Monitor monitor = mock(typeof(Monitor));
Input_Estimates estimates = Inputs.KnownEstimates(1000, 1000, 0, 0, 0, 0, 0); // we don't care about the rest of the estimates in this checking
// when
(new EstimationSanityChecker(formats, monitor)).SanityCheck(estimates);
// then
verifyNoMoreInteractions(monitor);
}
public DatabaseMigrator(MigrationProgressMonitor progressMonitor, FileSystemAbstraction fs, Config config, LogService logService, IndexProviderMap indexProviderMap, ExplicitIndexProvider indexProvider, PageCache pageCache, RecordFormats format, LogTailScanner tailScanner, JobScheduler jobScheduler)
{
this._progressMonitor = progressMonitor;
this._fs = fs;
this._config = config;
this._logService = logService;
this._indexProviderMap = indexProviderMap;
this._explicitIndexProvider = indexProvider;
this._pageCache = pageCache;
this._format = format;
this._tailScanner = tailScanner;
this._jobScheduler = jobScheduler;
}
public StoreFactory(DatabaseLayout databaseLayout, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, LogProvider logProvider, VersionContextSupplier versionContextSupplier, params OpenOption[] openOptions)
{
this._databaseLayout = databaseLayout;
this._config = config;
this._idGeneratorFactory = idGeneratorFactory;
this._fileSystemAbstraction = fileSystemAbstraction;
this._versionContextSupplier = versionContextSupplier;
this._recordFormats = recordFormats;
this._openOptions = openOptions;
(new RecordFormatPropertyConfigurator(recordFormats, config)).configure();
this._logProvider = logProvider;
this._pageCache = pageCache;
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldWarnAboutCounthigherThanCapacity()
public virtual void ShouldWarnAboutCounthigherThanCapacity()
{
// given
RecordFormats formats = Standard.LATEST_RECORD_FORMATS;
Monitor monitor = mock(typeof(Monitor));
Input_Estimates estimates = Inputs.KnownEstimates(formats.Node().MaxId * 2, formats.Relationship().MaxId * 2, 0, 0, 0, 0, 0); // we don't care about the rest of the estimates in this checking
// when
(new EstimationSanityChecker(formats, monitor)).SanityCheck(estimates);
// then
verify(monitor).mayExceedNodeIdCapacity(formats.Node().MaxId, estimates.NumberOfNodes());
verify(monitor).mayExceedRelationshipIdCapacity(formats.Relationship().MaxId, estimates.NumberOfRelationships());
}
public ParallelBatchImporter(DatabaseLayout directoryStructure, FileSystemAbstraction fileSystem, PageCache externalPageCache, Configuration config, LogService logService, ExecutionMonitor executionMonitor, AdditionalInitialIds additionalInitialIds, Config dbConfig, RecordFormats recordFormats, ImportLogic.Monitor monitor, JobScheduler jobScheduler)
{
this._externalPageCache = externalPageCache;
this._directoryStructure = directoryStructure;
this._fileSystem = fileSystem;
this._config = config;
this._logService = logService;
this._dbConfig = dbConfig;
this._recordFormats = recordFormats;
this._executionMonitor = executionMonitor;
this._additionalInitialIds = additionalInitialIds;
this._monitor = monitor;
this._jobScheduler = jobScheduler;
}
public override void Migrate(DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressReporter, string versionToMigrateFrom, string versionToMigrateTo)
{
RecordFormats from = RecordFormatSelector.selectForVersion(versionToMigrateFrom);
RecordFormats to = RecordFormatSelector.selectForVersion(versionToMigrateTo);
if (!from.HasCompatibleCapabilities(to, CapabilityType.INDEX))
{
_schemaIndexDirectory = _indexProvider.directoryStructure().rootDirectory();
if (_schemaIndexDirectory != null)
{
_deleteObsoleteIndexes = true;
}
// else this schema index provider doesn't have any persistent storage to delete.
}
}
private BatchingNeoStores(FileSystemAbstraction fileSystem, PageCache pageCache, File databaseDirectory, RecordFormats recordFormats, Config neo4jConfig, Configuration importConfiguration, LogService logService, AdditionalInitialIds initialIds, bool externalPageCache, IoTracer ioTracer)
{
this._fileSystem = fileSystem;
this._recordFormats = recordFormats;
this._importConfiguration = importConfiguration;
this._initialIds = initialIds;
this._logProvider = logService.InternalLogProvider;
this._databaseLayout = DatabaseLayout.of(databaseDirectory);
this._temporaryDatabaseLayout = DatabaseLayout.of(_databaseLayout.file(TEMP_STORE_NAME), TEMP_STORE_NAME);
this._neo4jConfig = neo4jConfig;
this._pageCache = pageCache;
this._ioTracer = ioTracer;
this._externalPageCache = externalPageCache;
this._idGeneratorFactory = new DefaultIdGeneratorFactory(fileSystem);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldNotDecideToAllocateDoubleRelationshipRecordUnitsonLargeAmountOfRelationshipsOnUnsupportedFormat() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldNotDecideToAllocateDoubleRelationshipRecordUnitsonLargeAmountOfRelationshipsOnUnsupportedFormat()
{
// given
RecordFormats formats = LATEST_RECORD_FORMATS;
using (BatchingNeoStores stores = BatchingNeoStores.BatchingNeoStoresWithExternalPageCache(Storage.fileSystem(), Storage.pageCache(), PageCacheTracer.NULL, Storage.directory().absolutePath(), formats, DEFAULT, NullLogService.Instance, EMPTY, Config.defaults()))
{
stores.CreateNew();
Input_Estimates estimates = Inputs.knownEstimates(0, DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD << 1, 0, 0, 0, 0, 0);
// when
bool doubleUnits = stores.DetermineDoubleRelationshipRecordUnits(estimates);
// then
assertFalse(doubleUnits);
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldRespectDbConfig() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldRespectDbConfig()
{
// GIVEN
int size = 10;
Config config = Config.defaults(stringMap(GraphDatabaseSettings.array_block_size.name(), size.ToString(), GraphDatabaseSettings.string_block_size.name(), size.ToString()));
// WHEN
RecordFormats recordFormats = LATEST_RECORD_FORMATS;
int headerSize = recordFormats.Dynamic().RecordHeaderSize;
using (JobScheduler jobScheduler = new ThreadPoolJobScheduler(), BatchingNeoStores store = BatchingNeoStores.BatchingNeoStoresConflict(Storage.fileSystem(), Storage.directory().absolutePath(), recordFormats, DEFAULT, NullLogService.Instance, EMPTY, config, jobScheduler))
{
store.CreateNew();
// THEN
assertEquals(size + headerSize, store.PropertyStore.ArrayStore.RecordSize);
assertEquals(size + headerSize, store.PropertyStore.StringStore.RecordSize);
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldNotOpenStoreWithNodesOrRelationshipsInIt() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldNotOpenStoreWithNodesOrRelationshipsInIt()
{
// GIVEN
SomeDataInTheDatabase();
// WHEN
try
{
using (JobScheduler jobScheduler = new ThreadPoolJobScheduler())
{
RecordFormats recordFormats = RecordFormatSelector.selectForConfig(Config.defaults(), NullLogProvider.Instance);
using (BatchingNeoStores store = BatchingNeoStores.BatchingNeoStoresConflict(Storage.fileSystem(), Storage.directory().databaseDir(), recordFormats, DEFAULT, NullLogService.Instance, EMPTY, Config.defaults(), jobScheduler))
{
store.CreateNew();
fail("Should fail on existing data");
}
}
}
catch (System.InvalidOperationException e)
{
// THEN
assertThat(e.Message, containsString("already contains"));
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public void migrate(org.neo4j.io.layout.DatabaseLayout directoryLayout, org.neo4j.io.layout.DatabaseLayout migrationLayout, org.neo4j.kernel.impl.util.monitoring.ProgressReporter progressMonitor, String versionToMigrateFrom, String versionToMigrateTo) throws java.io.IOException
public override void Migrate(DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressMonitor, string versionToMigrateFrom, string versionToMigrateTo)
{
IndexImplementation indexImplementation = _explicitIndexProvider.getProviderByName(LUCENE_EXPLICIT_INDEX_PROVIDER_NAME);
if (indexImplementation != null)
{
RecordFormats from = RecordFormatSelector.selectForVersion(versionToMigrateFrom);
RecordFormats to = RecordFormatSelector.selectForVersion(versionToMigrateTo);
if (!from.HasCompatibleCapabilities(to, CapabilityType.INDEX))
{
_originalExplicitIndexesRoot = indexImplementation.GetIndexImplementationDirectory(directoryLayout);
_migrationExplicitIndexesRoot = indexImplementation.GetIndexImplementationDirectory(migrationLayout);
if (IsNotEmptyDirectory(_originalExplicitIndexesRoot))
{
MigrateExplicitIndexes(progressMonitor);
_explicitIndexMigrated = true;
}
}
}
else
{
_log.debug("Lucene index provider not found, nothing to migrate.");
}
}
public override BatchImporter Instantiate(DatabaseLayout directoryStructure, FileSystemAbstraction fileSystem, PageCache externalPageCache, Configuration config, LogService logService, ExecutionMonitor executionMonitor, AdditionalInitialIds additionalInitialIds, Config dbConfig, RecordFormats recordFormats, ImportLogic.Monitor monitor, JobScheduler scheduler)
{
return(new ParallelBatchImporter(directoryStructure, fileSystem, externalPageCache, config, logService, executionMonitor, additionalInitialIds, dbConfig, recordFormats, monitor, scheduler));
}
internal virtual void SanityCheck([email protected]_Estimates inputEstimates, RecordFormats recordFormats, [email protected]_Visitable baseMemory, params [email protected]_Visitable[] memoryVisitables) { // At this point in time the store hasn't started so it won't show up in free memory reported from OS, // i.e. we have to include it here in the calculations. long estimatedCacheSize = estimatedCacheSize(baseMemory, memoryVisitables); long freeMemory = _freeMemoryLookup.AsLong; long optimalMinimalHeapSize = optimalMinimalHeapSize(inputEstimates, recordFormats); long actualHeapSize = _actualHeapSizeLookup.AsLong; bool freeMemoryIsKnown = freeMemory != VALUE_UNAVAILABLE; // Check if there's enough memory for the import if (freeMemoryIsKnown && actualHeapSize + freeMemory < estimatedCacheSize + optimalMinimalHeapSize) { _monitor.insufficientAvailableMemory(estimatedCacheSize, optimalMinimalHeapSize, freeMemory); return; // there's likely not available memory, no need to warn about anything else } // Check if the heap is big enough to handle the import if (actualHeapSize < optimalMinimalHeapSize) { _monitor.insufficientHeapSize(optimalMinimalHeapSize, actualHeapSize); return; // user have been warned about heap size issue } // Check if heap size could be tweaked if ((!freeMemoryIsKnown || freeMemory < estimatedCacheSize) && actualHeapSize > optimalMinimalHeapSize * 1.2) { _monitor.abundantHeapSize(optimalMinimalHeapSize, actualHeapSize); } }
/// <summary>
/// Calculates optimal and minimal heap size for an import. A minimal heap for an import has enough room for some amount
/// of working memory and the part of the page cache meta data living in the heap.
///
/// At the time of writing this the heap size is really only a function of store size, where parts of the page cache
/// meta data lives in the heap. For reference page cache meta data of a store of ~18TiB takes up ~10GiB of heap,
/// so pageCacheHeapUsage ~= storeSize / 2000. On top of that there must be some good old working memory of ~1-2 GiB
/// for handling objects created and operating during the import.
/// </summary>
/// <param name="estimates"> input estimates. </param>
/// <param name="recordFormats"> <seealso cref="RecordFormats"/>, containing record sizes. </param>
/// <returns> an optimal minimal heap size to use for this import. </returns>
public static long OptimalMinimalHeapSize([email protected]_Estimates estimates, RecordFormats recordFormats)
{
long estimatedStoreSize = estimatedStoreSize(estimates, recordFormats);
return(gibiBytes(1) + estimatedStoreSize / 2_000);
}
public EstimationSanityChecker(RecordFormats formats, ImportLogic.Monitor monitor)
{
this._formats = formats;
this._monitor = monitor;
}
public static BatchingNeoStores BatchingNeoStoresWithExternalPageCache(FileSystemAbstraction fileSystem, PageCache pageCache, PageCacheTracer tracer, File storeDir, RecordFormats recordFormats, Configuration config, LogService logService, AdditionalInitialIds initialIds, Config dbConfig)
{
Config neo4jConfig = GetNeo4jConfig(config, dbConfig);
return(new BatchingNeoStores(fileSystem, pageCache, storeDir, recordFormats, neo4jConfig, config, logService, initialIds, true, tracer.bytesWritten));
}
//JAVA TO C# CONVERTER NOTE: Members cannot have the same name as their enclosing type:
public static BatchingNeoStores BatchingNeoStoresConflict(FileSystemAbstraction fileSystem, File storeDir, RecordFormats recordFormats, Configuration config, LogService logService, AdditionalInitialIds initialIds, Config dbConfig, JobScheduler jobScheduler)
{
Config neo4jConfig = GetNeo4jConfig(config, dbConfig);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.neo4j.io.pagecache.tracing.PageCacheTracer tracer = new org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer();
PageCacheTracer tracer = new DefaultPageCacheTracer();
PageCache pageCache = CreatePageCache(fileSystem, neo4jConfig, logService.InternalLogProvider, tracer, DefaultPageCursorTracerSupplier.INSTANCE, EmptyVersionContextSupplier.EMPTY, jobScheduler);
return(new BatchingNeoStores(fileSystem, pageCache, storeDir, recordFormats, neo4jConfig, config, logService, initialIds, false, tracer.bytesWritten));
}
public static long EstimatedStoreSize([email protected]_Estimates estimates, RecordFormats recordFormats) { long nodeSize = estimates.NumberOfNodes() * recordFormats.Node().getRecordSize(NO_STORE_HEADER); long relationshipSize = estimates.NumberOfRelationships() * recordFormats.Relationship().getRecordSize(NO_STORE_HEADER); long propertySize = estimates.SizeOfNodeProperties() + estimates.SizeOfRelationshipProperties(); long tempIdPropertySize = estimates.NumberOfNodes() * recordFormats.Property().getRecordSize(NO_STORE_HEADER); return(DefensivelyPadMemoryEstimate(nodeSize + relationshipSize + propertySize + tempIdPropertySize)); }
public StandardToEnterpriseStoreUpgraderTest(RecordFormats recordFormats) : base(recordFormats)
{
}
