public NodeImporter(BatchingNeoStores stores, IdMapper idMapper, Monitor monitor) : base(stores, monitor) { this._labelTokenRepository = stores.LabelRepository; this._idMapper = idMapper; this._nodeStore = stores.NodeStore; this._nodeRecord = _nodeStore.newRecord(); this._nodeIds = new BatchingIdGetter(_nodeStore); this._idPropertyStore = stores.TemporaryPropertyStore; this._idPropertyRecord = _idPropertyStore.newRecord(); _nodeRecord.InUse = true; }
private void InitializeMisc(BatchingNeoStores neoStores, DataStatistics distribution) { PrintStageHeader("(4/4) Post processing", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores))); long actualNodeCount = distribution.NodeCount; // The reason the highId of the relationship store is used, as opposed to actual number of imported relationships // is that the stages underneath operate on id ranges, not knowing which records are actually in use. long relationshipRecordIdCount = neoStores.RelationshipStore.HighId; long groupCount = neoStores.TemporaryRelationshipGroupStore.HighId; InitializeProgress(groupCount + groupCount + groupCount + actualNodeCount + relationshipRecordIdCount, ImportStage.PostProcessing); }
private void InitializeLinking(BatchingNeoStores neoStores, NodeRelationshipCache nodeRelationshipCache, DataStatistics distribution) { PrintStageHeader("(3/4) Relationship linking", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + defensivelyPadMemoryEstimate(nodeRelationshipCache.MemoryEstimation(distribution.NodeCount)))); // The reason the highId of the relationship store is used, as opposed to actual number of imported relationships // is that the stages underneath operate on id ranges, not knowing which records are actually in use. long relationshipRecordIdCount = neoStores.RelationshipStore.HighId; // The progress counting of linking stages is special anyway, in that it uses the "progress" stats key, // which is based on actual number of relationships, not relationship ids. long actualRelationshipCount = distribution.RelationshipCount; InitializeProgress(relationshipRecordIdCount + actualRelationshipCount * 2 + actualRelationshipCount * 2, ImportStage.Linking); }
protected internal EntityImporter(BatchingNeoStores stores, Monitor monitor) { this._propertyStore = stores.PropertyStore; this._propertyKeyTokenRepository = stores.PropertyKeyRepository; this.Monitor = monitor; for (int i = 0; i < _propertyBlocks.Length; i++) { _propertyBlocks[i] = new PropertyBlock(); } this._propertyRecord = _propertyStore.newRecord(); this._propertyIds = new BatchingIdGetter(_propertyStore); this._dynamicStringRecordAllocator = new StandardDynamicRecordAllocator(new BatchingIdGetter(_propertyStore.StringStore, _propertyStore.StringStore.RecordsPerPage), _propertyStore.StringStore.RecordDataSize); this._dynamicArrayRecordAllocator = new StandardDynamicRecordAllocator(new BatchingIdGetter(_propertyStore.ArrayStore, _propertyStore.ArrayStore.RecordsPerPage), _propertyStore.StringStore.RecordDataSize); }
public override void Initialize(DependencyResolver dependencyResolver) { this._dependencyResolver = dependencyResolver; Input_Estimates estimates = dependencyResolver.ResolveDependency(typeof(Input_Estimates)); BatchingNeoStores neoStores = dependencyResolver.ResolveDependency(typeof(BatchingNeoStores)); IdMapper idMapper = dependencyResolver.ResolveDependency(typeof(IdMapper)); NodeRelationshipCache nodeRelationshipCache = dependencyResolver.ResolveDependency(typeof(NodeRelationshipCache)); _pageCacheArrayFactoryMonitor = dependencyResolver.ResolveDependency(typeof(PageCacheArrayFactoryMonitor)); long biggestCacheMemory = estimatedCacheSize(neoStores, nodeRelationshipCache.MemoryEstimation(estimates.NumberOfNodes()), idMapper.MemoryEstimation(estimates.NumberOfNodes())); PrintStageHeader("Import starting", ESTIMATED_NUMBER_OF_NODES, count(estimates.NumberOfNodes()), ESTIMATED_NUMBER_OF_NODE_PROPERTIES, count(estimates.NumberOfNodeProperties()), ESTIMATED_NUMBER_OF_RELATIONSHIPS, count(estimates.NumberOfRelationships()), ESTIMATED_NUMBER_OF_RELATIONSHIP_PROPERTIES, count(estimates.NumberOfRelationshipProperties()), ESTIMATED_DISK_SPACE_USAGE, bytes(NodesDiskUsage(estimates, neoStores) + RelationshipsDiskUsage(estimates, neoStores) + estimates.SizeOfNodeProperties() + estimates.SizeOfRelationshipProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(biggestCacheMemory)); Console.WriteLine(); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void closeImporterWithoutDiagnosticState() throws java.io.IOException //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void CloseImporterWithoutDiagnosticState() { ExecutionMonitor monitor = mock(typeof(ExecutionMonitor)); using (BatchingNeoStores stores = batchingNeoStoresWithExternalPageCache(Storage.fileSystem(), Storage.pageCache(), NULL, Storage.directory().directory(), defaultFormat(), DEFAULT, Instance, EMPTY, defaults())) { //noinspection EmptyTryBlock using (ImportLogic logic = new ImportLogic(Storage.directory().directory(), Storage.fileSystem(), stores, DEFAULT, Instance, monitor, defaultFormat(), NO_MONITOR)) { // nothing to run in this import logic.Success(); } } verify(monitor).done(eq(true), anyLong(), contains("Data statistics is not available.")); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldUseDataStatisticsCountsForPrintingFinalStats() throws java.io.IOException //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldUseDataStatisticsCountsForPrintingFinalStats() { // given ExecutionMonitor monitor = mock(typeof(ExecutionMonitor)); using (BatchingNeoStores stores = batchingNeoStoresWithExternalPageCache(Storage.fileSystem(), Storage.pageCache(), NULL, Storage.directory().directory(), defaultFormat(), DEFAULT, Instance, EMPTY, defaults())) { // when RelationshipTypeCount[] relationshipTypeCounts = new RelationshipTypeCount[] { new RelationshipTypeCount(0, 33), new RelationshipTypeCount(1, 66) }; DataStatistics dataStatistics = new DataStatistics(100123, 100456, relationshipTypeCounts); using (ImportLogic logic = new ImportLogic(Storage.directory().directory(), Storage.fileSystem(), stores, DEFAULT, Instance, monitor, defaultFormat(), NO_MONITOR)) { logic.PutState(dataStatistics); logic.Success(); } // then verify(monitor).done(eq(true), anyLong(), contains(dataStatistics.ToString())); } }
private void InitializeRelationshipImport(Input_Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores) { long numberOfRelationships = estimates.NumberOfRelationships(); PrintStageHeader("(2/4) Relationship import", ESTIMATED_NUMBER_OF_RELATIONSHIPS, count(numberOfRelationships), ESTIMATED_DISK_SPACE_USAGE, bytes(RelationshipsDiskUsage(estimates, neoStores) + estimates.SizeOfRelationshipProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + totalMemoryUsageOf(idMapper))); InitializeProgress(numberOfRelationships, ImportStage.RelationshipImport); }
private void InitializeNodeImport(Input_Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores) { long numberOfNodes = estimates.NumberOfNodes(); PrintStageHeader("(1/4) Node import", ESTIMATED_NUMBER_OF_NODES, count(numberOfNodes), ESTIMATED_DISK_SPACE_USAGE, bytes(NodesDiskUsage(estimates, neoStores) + estimates.SizeOfNodeProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + defensivelyPadMemoryEstimate(idMapper.MemoryEstimation(numberOfNodes)))); // A difficulty with the goal here is that we don't know how much work there is to be done in id mapper preparation stage. // In addition to nodes themselves and SPLIT,SORT,DETECT there may be RESOLVE,SORT,DEDUPLICATE too, if there are collisions long goal = idMapper.NeedsPreparation() ? numberOfNodes + Weighted(IdMapperPreparationStage.NAME, numberOfNodes * 4) : numberOfNodes; InitializeProgress(goal, ImportStage.NodeImport); }
private static long RelationshipsDiskUsage(Input_Estimates estimates, BatchingNeoStores neoStores) { return(estimates.NumberOfRelationships() * neoStores.RelationshipStore.RecordSize * (neoStores.UsesDoubleRelationshipRecordUnits() ? 2 : 1)); }
private static long NodesDiskUsage(Input_Estimates estimates, BatchingNeoStores neoStores) { return(estimates.NumberOfNodes() * neoStores.NodeStore.RecordSize + estimates.NumberOfNodeLabels()); }
private static long BaselineMemoryRequirement(BatchingNeoStores neoStores) { return(totalMemoryUsageOf(neoStores)); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Before public void start() throws java.io.IOException //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void Start() { _jobScheduler = new ThreadPoolJobScheduler(); _stores = BatchingNeoStores.batchingNeoStores(_fileSystemRule.get(), _directory.absolutePath(), Format, _config, NullLogService.Instance, AdditionalInitialIds.EMPTY, Config.defaults(), _jobScheduler); _stores.createNew(); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public static DataStatistics importRelationships(int numRunners, org.neo4j.unsafe.impl.batchimport.input.Input input, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper idMapper, org.neo4j.unsafe.impl.batchimport.input.Collector badCollector, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, Monitor monitor, boolean validateRelationshipData) throws java.io.IOException public static DataStatistics ImportRelationships(int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, Collector badCollector, ExecutionMonitor executionMonitor, Monitor monitor, bool validateRelationshipData) { DataStatistics typeDistribution = new DataStatistics(monitor, new RelationshipTypeCount[0]); System.Func <EntityImporter> importers = () => new RelationshipImporter(stores, idMapper, typeDistribution, monitor, badCollector, validateRelationshipData, stores.UsesDoubleRelationshipRecordUnits()); ImportData(RELATIONSHIP_IMPORT_NAME, numRunners, input.Relationships(), stores, importers, executionMonitor, new MemoryUsageStatsProvider(stores, idMapper)); return(typeDistribution); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public static void importNodes(int numRunners, org.neo4j.unsafe.impl.batchimport.input.Input input, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper idMapper, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, Monitor monitor) throws java.io.IOException public static void ImportNodes(int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, ExecutionMonitor executionMonitor, Monitor monitor) { System.Func <EntityImporter> importers = () => new NodeImporter(stores, idMapper, monitor); ImportData(NODE_IMPORT_NAME, numRunners, input.Nodes(), stores, importers, executionMonitor, new MemoryUsageStatsProvider(stores, idMapper)); }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: private static long importData(String title, int numRunners, InputIterable data, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, System.Func<EntityImporter> visitors, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, org.neo4j.unsafe.impl.batchimport.stats.StatsProvider memoryStatsProvider) throws java.io.IOException private static long ImportData(string title, int numRunners, InputIterable data, BatchingNeoStores stores, System.Func <EntityImporter> visitors, ExecutionMonitor executionMonitor, StatsProvider memoryStatsProvider) { LongAdder roughEntityCountProgress = new LongAdder(); ExecutorService pool = Executors.newFixedThreadPool(numRunners, new NamedThreadFactory(title + "Importer")); IoMonitor writeMonitor = new IoMonitor(stores.IoTracer); ControllableStep step = new ControllableStep(title, roughEntityCountProgress, Configuration.DEFAULT, writeMonitor, memoryStatsProvider); StageExecution execution = new StageExecution(title, null, Configuration.DEFAULT, Collections.singletonList(step), 0); long startTime = currentTimeMillis(); using (InputIterator dataIterator = data.GetEnumerator()) { executionMonitor.Start(execution); for (int i = 0; i < numRunners; i++) { pool.submit(new ExhaustingEntityImporterRunnable(execution, dataIterator, visitors(), roughEntityCountProgress)); } pool.shutdown(); long nextWait = 0; try { while (!pool.awaitTermination(nextWait, TimeUnit.MILLISECONDS)) { executionMonitor.Check(execution); nextWait = executionMonitor.NextCheckTime() - currentTimeMillis(); } } catch (InterruptedException e) { Thread.CurrentThread.Interrupt(); throw new IOException(e); } } execution.AssertHealthy(); step.MarkAsCompleted(); writeMonitor.Stop(); executionMonitor.End(execution, currentTimeMillis() - startTime); execution.AssertHealthy(); return(roughEntityCountProgress.sum()); }