private StoreFactory StoreFactory(Config config, params OpenOption[] openOptions) { LogProvider logProvider = NullLogProvider.Instance; DatabaseLayout databaseLayout = _testDirectory.databaseLayout(); RecordFormats recordFormats = selectForStoreOrConfig(config, databaseLayout, _fsRule, _pageCache, logProvider); return(new StoreFactory(databaseLayout, config, _idGeneratorFactory, _pageCache, _fsRule.get(), recordFormats, logProvider, EmptyVersionContextSupplier.EMPTY, openOptions)); }
private StoreFactory GetStoreFactory(DatabaseLayout directoryStructure, string versionToMigrateFrom) { NullLogProvider logProvider = NullLogProvider.Instance; RecordFormats recordFormats = selectForVersion(versionToMigrateFrom); IdGeneratorFactory idGeneratorFactory = new ReadOnlyIdGeneratorFactory(_fileSystem); return(new StoreFactory(directoryStructure, _config, idGeneratorFactory, _pageCache, _fileSystem, recordFormats, logProvider, EmptyVersionContextSupplier.EMPTY)); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldNotWantIfCountWayLowerThanCapacity() public virtual void ShouldNotWantIfCountWayLowerThanCapacity() { // given RecordFormats formats = Standard.LATEST_RECORD_FORMATS; Monitor monitor = mock(typeof(Monitor)); Input_Estimates estimates = Inputs.KnownEstimates(1000, 1000, 0, 0, 0, 0, 0); // we don't care about the rest of the estimates in this checking // when (new EstimationSanityChecker(formats, monitor)).SanityCheck(estimates); // then verifyNoMoreInteractions(monitor); }
public DatabaseMigrator(MigrationProgressMonitor progressMonitor, FileSystemAbstraction fs, Config config, LogService logService, IndexProviderMap indexProviderMap, ExplicitIndexProvider indexProvider, PageCache pageCache, RecordFormats format, LogTailScanner tailScanner, JobScheduler jobScheduler) { this._progressMonitor = progressMonitor; this._fs = fs; this._config = config; this._logService = logService; this._indexProviderMap = indexProviderMap; this._explicitIndexProvider = indexProvider; this._pageCache = pageCache; this._format = format; this._tailScanner = tailScanner; this._jobScheduler = jobScheduler; }
public StoreFactory(DatabaseLayout databaseLayout, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, LogProvider logProvider, VersionContextSupplier versionContextSupplier, params OpenOption[] openOptions) { this._databaseLayout = databaseLayout; this._config = config; this._idGeneratorFactory = idGeneratorFactory; this._fileSystemAbstraction = fileSystemAbstraction; this._versionContextSupplier = versionContextSupplier; this._recordFormats = recordFormats; this._openOptions = openOptions; (new RecordFormatPropertyConfigurator(recordFormats, config)).configure(); this._logProvider = logProvider; this._pageCache = pageCache; }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldWarnAboutCounthigherThanCapacity() public virtual void ShouldWarnAboutCounthigherThanCapacity() { // given RecordFormats formats = Standard.LATEST_RECORD_FORMATS; Monitor monitor = mock(typeof(Monitor)); Input_Estimates estimates = Inputs.KnownEstimates(formats.Node().MaxId * 2, formats.Relationship().MaxId * 2, 0, 0, 0, 0, 0); // we don't care about the rest of the estimates in this checking // when (new EstimationSanityChecker(formats, monitor)).SanityCheck(estimates); // then verify(monitor).mayExceedNodeIdCapacity(formats.Node().MaxId, estimates.NumberOfNodes()); verify(monitor).mayExceedRelationshipIdCapacity(formats.Relationship().MaxId, estimates.NumberOfRelationships()); }
public ParallelBatchImporter(DatabaseLayout directoryStructure, FileSystemAbstraction fileSystem, PageCache externalPageCache, Configuration config, LogService logService, ExecutionMonitor executionMonitor, AdditionalInitialIds additionalInitialIds, Config dbConfig, RecordFormats recordFormats, ImportLogic.Monitor monitor, JobScheduler jobScheduler) { this._externalPageCache = externalPageCache; this._directoryStructure = directoryStructure; this._fileSystem = fileSystem; this._config = config; this._logService = logService; this._dbConfig = dbConfig; this._recordFormats = recordFormats; this._executionMonitor = executionMonitor; this._additionalInitialIds = additionalInitialIds; this._monitor = monitor; this._jobScheduler = jobScheduler; }
public override void Migrate(DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressReporter, string versionToMigrateFrom, string versionToMigrateTo) { RecordFormats from = RecordFormatSelector.selectForVersion(versionToMigrateFrom); RecordFormats to = RecordFormatSelector.selectForVersion(versionToMigrateTo); if (!from.HasCompatibleCapabilities(to, CapabilityType.INDEX)) { _schemaIndexDirectory = _indexProvider.directoryStructure().rootDirectory(); if (_schemaIndexDirectory != null) { _deleteObsoleteIndexes = true; } // else this schema index provider doesn't have any persistent storage to delete. } }
private BatchingNeoStores(FileSystemAbstraction fileSystem, PageCache pageCache, File databaseDirectory, RecordFormats recordFormats, Config neo4jConfig, Configuration importConfiguration, LogService logService, AdditionalInitialIds initialIds, bool externalPageCache, IoTracer ioTracer) { this._fileSystem = fileSystem; this._recordFormats = recordFormats; this._importConfiguration = importConfiguration; this._initialIds = initialIds; this._logProvider = logService.InternalLogProvider; this._databaseLayout = DatabaseLayout.of(databaseDirectory); this._temporaryDatabaseLayout = DatabaseLayout.of(_databaseLayout.file(TEMP_STORE_NAME), TEMP_STORE_NAME); this._neo4jConfig = neo4jConfig; this._pageCache = pageCache; this._ioTracer = ioTracer; this._externalPageCache = externalPageCache; this._idGeneratorFactory = new DefaultIdGeneratorFactory(fileSystem); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldNotDecideToAllocateDoubleRelationshipRecordUnitsonLargeAmountOfRelationshipsOnUnsupportedFormat() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldNotDecideToAllocateDoubleRelationshipRecordUnitsonLargeAmountOfRelationshipsOnUnsupportedFormat() { // given RecordFormats formats = LATEST_RECORD_FORMATS; using (BatchingNeoStores stores = BatchingNeoStores.BatchingNeoStoresWithExternalPageCache(Storage.fileSystem(), Storage.pageCache(), PageCacheTracer.NULL, Storage.directory().absolutePath(), formats, DEFAULT, NullLogService.Instance, EMPTY, Config.defaults())) { stores.CreateNew(); Input_Estimates estimates = Inputs.knownEstimates(0, DOUBLE_RELATIONSHIP_RECORD_UNIT_THRESHOLD << 1, 0, 0, 0, 0, 0); // when bool doubleUnits = stores.DetermineDoubleRelationshipRecordUnits(estimates); // then assertFalse(doubleUnits); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldRespectDbConfig() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldRespectDbConfig() { // GIVEN int size = 10; Config config = Config.defaults(stringMap(GraphDatabaseSettings.array_block_size.name(), size.ToString(), GraphDatabaseSettings.string_block_size.name(), size.ToString())); // WHEN RecordFormats recordFormats = LATEST_RECORD_FORMATS; int headerSize = recordFormats.Dynamic().RecordHeaderSize; using (JobScheduler jobScheduler = new ThreadPoolJobScheduler(), BatchingNeoStores store = BatchingNeoStores.BatchingNeoStoresConflict(Storage.fileSystem(), Storage.directory().absolutePath(), recordFormats, DEFAULT, NullLogService.Instance, EMPTY, config, jobScheduler)) { store.CreateNew(); // THEN assertEquals(size + headerSize, store.PropertyStore.ArrayStore.RecordSize); assertEquals(size + headerSize, store.PropertyStore.StringStore.RecordSize); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldNotOpenStoreWithNodesOrRelationshipsInIt() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldNotOpenStoreWithNodesOrRelationshipsInIt() { // GIVEN SomeDataInTheDatabase(); // WHEN try { using (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) { RecordFormats recordFormats = RecordFormatSelector.selectForConfig(Config.defaults(), NullLogProvider.Instance); using (BatchingNeoStores store = BatchingNeoStores.BatchingNeoStoresConflict(Storage.fileSystem(), Storage.directory().databaseDir(), recordFormats, DEFAULT, NullLogService.Instance, EMPTY, Config.defaults(), jobScheduler)) { store.CreateNew(); fail("Should fail on existing data"); } } } catch (System.InvalidOperationException e) { // THEN assertThat(e.Message, containsString("already contains")); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public void migrate(org.neo4j.io.layout.DatabaseLayout directoryLayout, org.neo4j.io.layout.DatabaseLayout migrationLayout, org.neo4j.kernel.impl.util.monitoring.ProgressReporter progressMonitor, String versionToMigrateFrom, String versionToMigrateTo) throws java.io.IOException public override void Migrate(DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressMonitor, string versionToMigrateFrom, string versionToMigrateTo) { IndexImplementation indexImplementation = _explicitIndexProvider.getProviderByName(LUCENE_EXPLICIT_INDEX_PROVIDER_NAME); if (indexImplementation != null) { RecordFormats from = RecordFormatSelector.selectForVersion(versionToMigrateFrom); RecordFormats to = RecordFormatSelector.selectForVersion(versionToMigrateTo); if (!from.HasCompatibleCapabilities(to, CapabilityType.INDEX)) { _originalExplicitIndexesRoot = indexImplementation.GetIndexImplementationDirectory(directoryLayout); _migrationExplicitIndexesRoot = indexImplementation.GetIndexImplementationDirectory(migrationLayout); if (IsNotEmptyDirectory(_originalExplicitIndexesRoot)) { MigrateExplicitIndexes(progressMonitor); _explicitIndexMigrated = true; } } } else { _log.debug("Lucene index provider not found, nothing to migrate."); } }
public override BatchImporter Instantiate(DatabaseLayout directoryStructure, FileSystemAbstraction fileSystem, PageCache externalPageCache, Configuration config, LogService logService, ExecutionMonitor executionMonitor, AdditionalInitialIds additionalInitialIds, Config dbConfig, RecordFormats recordFormats, ImportLogic.Monitor monitor, JobScheduler scheduler) { return(new ParallelBatchImporter(directoryStructure, fileSystem, externalPageCache, config, logService, executionMonitor, additionalInitialIds, dbConfig, recordFormats, monitor, scheduler)); }
internal virtual void SanityCheck([email protected]_Estimates inputEstimates, RecordFormats recordFormats, [email protected]_Visitable baseMemory, params [email protected]_Visitable[] memoryVisitables) { // At this point in time the store hasn't started so it won't show up in free memory reported from OS, // i.e. we have to include it here in the calculations. long estimatedCacheSize = estimatedCacheSize(baseMemory, memoryVisitables); long freeMemory = _freeMemoryLookup.AsLong; long optimalMinimalHeapSize = optimalMinimalHeapSize(inputEstimates, recordFormats); long actualHeapSize = _actualHeapSizeLookup.AsLong; bool freeMemoryIsKnown = freeMemory != VALUE_UNAVAILABLE; // Check if there's enough memory for the import if (freeMemoryIsKnown && actualHeapSize + freeMemory < estimatedCacheSize + optimalMinimalHeapSize) { _monitor.insufficientAvailableMemory(estimatedCacheSize, optimalMinimalHeapSize, freeMemory); return; // there's likely not available memory, no need to warn about anything else } // Check if the heap is big enough to handle the import if (actualHeapSize < optimalMinimalHeapSize) { _monitor.insufficientHeapSize(optimalMinimalHeapSize, actualHeapSize); return; // user have been warned about heap size issue } // Check if heap size could be tweaked if ((!freeMemoryIsKnown || freeMemory < estimatedCacheSize) && actualHeapSize > optimalMinimalHeapSize * 1.2) { _monitor.abundantHeapSize(optimalMinimalHeapSize, actualHeapSize); } }
/// <summary> /// Calculates optimal and minimal heap size for an import. A minimal heap for an import has enough room for some amount /// of working memory and the part of the page cache meta data living in the heap. /// /// At the time of writing this the heap size is really only a function of store size, where parts of the page cache /// meta data lives in the heap. For reference page cache meta data of a store of ~18TiB takes up ~10GiB of heap, /// so pageCacheHeapUsage ~= storeSize / 2000. On top of that there must be some good old working memory of ~1-2 GiB /// for handling objects created and operating during the import. /// </summary> /// <param name="estimates"> input estimates. </param> /// <param name="recordFormats"> <seealso cref="RecordFormats"/>, containing record sizes. </param> /// <returns> an optimal minimal heap size to use for this import. </returns> public static long OptimalMinimalHeapSize([email protected]_Estimates estimates, RecordFormats recordFormats) { long estimatedStoreSize = estimatedStoreSize(estimates, recordFormats); return(gibiBytes(1) + estimatedStoreSize / 2_000); }
public EstimationSanityChecker(RecordFormats formats, ImportLogic.Monitor monitor) { this._formats = formats; this._monitor = monitor; }
public static BatchingNeoStores BatchingNeoStoresWithExternalPageCache(FileSystemAbstraction fileSystem, PageCache pageCache, PageCacheTracer tracer, File storeDir, RecordFormats recordFormats, Configuration config, LogService logService, AdditionalInitialIds initialIds, Config dbConfig) { Config neo4jConfig = GetNeo4jConfig(config, dbConfig); return(new BatchingNeoStores(fileSystem, pageCache, storeDir, recordFormats, neo4jConfig, config, logService, initialIds, true, tracer.bytesWritten)); }
//JAVA TO C# CONVERTER NOTE: Members cannot have the same name as their enclosing type: public static BatchingNeoStores BatchingNeoStoresConflict(FileSystemAbstraction fileSystem, File storeDir, RecordFormats recordFormats, Configuration config, LogService logService, AdditionalInitialIds initialIds, Config dbConfig, JobScheduler jobScheduler) { Config neo4jConfig = GetNeo4jConfig(config, dbConfig); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final org.neo4j.io.pagecache.tracing.PageCacheTracer tracer = new org.neo4j.io.pagecache.tracing.DefaultPageCacheTracer(); PageCacheTracer tracer = new DefaultPageCacheTracer(); PageCache pageCache = CreatePageCache(fileSystem, neo4jConfig, logService.InternalLogProvider, tracer, DefaultPageCursorTracerSupplier.INSTANCE, EmptyVersionContextSupplier.EMPTY, jobScheduler); return(new BatchingNeoStores(fileSystem, pageCache, storeDir, recordFormats, neo4jConfig, config, logService, initialIds, false, tracer.bytesWritten)); }
public static long EstimatedStoreSize([email protected]_Estimates estimates, RecordFormats recordFormats) { long nodeSize = estimates.NumberOfNodes() * recordFormats.Node().getRecordSize(NO_STORE_HEADER); long relationshipSize = estimates.NumberOfRelationships() * recordFormats.Relationship().getRecordSize(NO_STORE_HEADER); long propertySize = estimates.SizeOfNodeProperties() + estimates.SizeOfRelationshipProperties(); long tempIdPropertySize = estimates.NumberOfNodes() * recordFormats.Property().getRecordSize(NO_STORE_HEADER); return(DefensivelyPadMemoryEstimate(nodeSize + relationshipSize + propertySize + tempIdPropertySize)); }
public StandardToEnterpriseStoreUpgraderTest(RecordFormats recordFormats) : base(recordFormats) { }