public NodeImporter(BatchingNeoStores stores, IdMapper idMapper, Monitor monitor) : base(stores, monitor)
{
this._labelTokenRepository = stores.LabelRepository;
this._idMapper = idMapper;
this._nodeStore = stores.NodeStore;
this._nodeRecord = _nodeStore.newRecord();
this._nodeIds = new BatchingIdGetter(_nodeStore);
this._idPropertyStore = stores.TemporaryPropertyStore;
this._idPropertyRecord = _idPropertyStore.newRecord();
_nodeRecord.InUse = true;
}
private void InitializeMisc(BatchingNeoStores neoStores, DataStatistics distribution)
{
PrintStageHeader("(4/4) Post processing", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores)));
long actualNodeCount = distribution.NodeCount;
// The reason the highId of the relationship store is used, as opposed to actual number of imported relationships
// is that the stages underneath operate on id ranges, not knowing which records are actually in use.
long relationshipRecordIdCount = neoStores.RelationshipStore.HighId;
long groupCount = neoStores.TemporaryRelationshipGroupStore.HighId;
InitializeProgress(groupCount + groupCount + groupCount + actualNodeCount + relationshipRecordIdCount, ImportStage.PostProcessing);
}
private void InitializeLinking(BatchingNeoStores neoStores, NodeRelationshipCache nodeRelationshipCache, DataStatistics distribution)
{
PrintStageHeader("(3/4) Relationship linking", ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + defensivelyPadMemoryEstimate(nodeRelationshipCache.MemoryEstimation(distribution.NodeCount))));
// The reason the highId of the relationship store is used, as opposed to actual number of imported relationships
// is that the stages underneath operate on id ranges, not knowing which records are actually in use.
long relationshipRecordIdCount = neoStores.RelationshipStore.HighId;
// The progress counting of linking stages is special anyway, in that it uses the "progress" stats key,
// which is based on actual number of relationships, not relationship ids.
long actualRelationshipCount = distribution.RelationshipCount;
InitializeProgress(relationshipRecordIdCount + actualRelationshipCount * 2 + actualRelationshipCount * 2, ImportStage.Linking);
}
protected internal EntityImporter(BatchingNeoStores stores, Monitor monitor)
{
this._propertyStore = stores.PropertyStore;
this._propertyKeyTokenRepository = stores.PropertyKeyRepository;
this.Monitor = monitor;
for (int i = 0; i < _propertyBlocks.Length; i++)
{
_propertyBlocks[i] = new PropertyBlock();
}
this._propertyRecord = _propertyStore.newRecord();
this._propertyIds = new BatchingIdGetter(_propertyStore);
this._dynamicStringRecordAllocator = new StandardDynamicRecordAllocator(new BatchingIdGetter(_propertyStore.StringStore, _propertyStore.StringStore.RecordsPerPage), _propertyStore.StringStore.RecordDataSize);
this._dynamicArrayRecordAllocator = new StandardDynamicRecordAllocator(new BatchingIdGetter(_propertyStore.ArrayStore, _propertyStore.ArrayStore.RecordsPerPage), _propertyStore.StringStore.RecordDataSize);
}
public override void Initialize(DependencyResolver dependencyResolver)
{
this._dependencyResolver = dependencyResolver;
Input_Estimates estimates = dependencyResolver.ResolveDependency(typeof(Input_Estimates));
BatchingNeoStores neoStores = dependencyResolver.ResolveDependency(typeof(BatchingNeoStores));
IdMapper idMapper = dependencyResolver.ResolveDependency(typeof(IdMapper));
NodeRelationshipCache nodeRelationshipCache = dependencyResolver.ResolveDependency(typeof(NodeRelationshipCache));
_pageCacheArrayFactoryMonitor = dependencyResolver.ResolveDependency(typeof(PageCacheArrayFactoryMonitor));
long biggestCacheMemory = estimatedCacheSize(neoStores, nodeRelationshipCache.MemoryEstimation(estimates.NumberOfNodes()), idMapper.MemoryEstimation(estimates.NumberOfNodes()));
PrintStageHeader("Import starting", ESTIMATED_NUMBER_OF_NODES, count(estimates.NumberOfNodes()), ESTIMATED_NUMBER_OF_NODE_PROPERTIES, count(estimates.NumberOfNodeProperties()), ESTIMATED_NUMBER_OF_RELATIONSHIPS, count(estimates.NumberOfRelationships()), ESTIMATED_NUMBER_OF_RELATIONSHIP_PROPERTIES, count(estimates.NumberOfRelationshipProperties()), ESTIMATED_DISK_SPACE_USAGE, bytes(NodesDiskUsage(estimates, neoStores) + RelationshipsDiskUsage(estimates, neoStores) + estimates.SizeOfNodeProperties() + estimates.SizeOfRelationshipProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(biggestCacheMemory));
Console.WriteLine();
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void closeImporterWithoutDiagnosticState() throws java.io.IOException
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void CloseImporterWithoutDiagnosticState()
{
ExecutionMonitor monitor = mock(typeof(ExecutionMonitor));
using (BatchingNeoStores stores = batchingNeoStoresWithExternalPageCache(Storage.fileSystem(), Storage.pageCache(), NULL, Storage.directory().directory(), defaultFormat(), DEFAULT, Instance, EMPTY, defaults()))
{
//noinspection EmptyTryBlock
using (ImportLogic logic = new ImportLogic(Storage.directory().directory(), Storage.fileSystem(), stores, DEFAULT, Instance, monitor, defaultFormat(), NO_MONITOR))
{
// nothing to run in this import
logic.Success();
}
}
verify(monitor).done(eq(true), anyLong(), contains("Data statistics is not available."));
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldUseDataStatisticsCountsForPrintingFinalStats() throws java.io.IOException
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldUseDataStatisticsCountsForPrintingFinalStats()
{
// given
ExecutionMonitor monitor = mock(typeof(ExecutionMonitor));
using (BatchingNeoStores stores = batchingNeoStoresWithExternalPageCache(Storage.fileSystem(), Storage.pageCache(), NULL, Storage.directory().directory(), defaultFormat(), DEFAULT, Instance, EMPTY, defaults()))
{
// when
RelationshipTypeCount[] relationshipTypeCounts = new RelationshipTypeCount[]
{
new RelationshipTypeCount(0, 33),
new RelationshipTypeCount(1, 66)
};
DataStatistics dataStatistics = new DataStatistics(100123, 100456, relationshipTypeCounts);
using (ImportLogic logic = new ImportLogic(Storage.directory().directory(), Storage.fileSystem(), stores, DEFAULT, Instance, monitor, defaultFormat(), NO_MONITOR))
{
logic.PutState(dataStatistics);
logic.Success();
}
// then
verify(monitor).done(eq(true), anyLong(), contains(dataStatistics.ToString()));
}
}
private void InitializeRelationshipImport(Input_Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores)
{
long numberOfRelationships = estimates.NumberOfRelationships();
PrintStageHeader("(2/4) Relationship import", ESTIMATED_NUMBER_OF_RELATIONSHIPS, count(numberOfRelationships), ESTIMATED_DISK_SPACE_USAGE, bytes(RelationshipsDiskUsage(estimates, neoStores) + estimates.SizeOfRelationshipProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + totalMemoryUsageOf(idMapper)));
InitializeProgress(numberOfRelationships, ImportStage.RelationshipImport);
}
private void InitializeNodeImport(Input_Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores)
{
long numberOfNodes = estimates.NumberOfNodes();
PrintStageHeader("(1/4) Node import", ESTIMATED_NUMBER_OF_NODES, count(numberOfNodes), ESTIMATED_DISK_SPACE_USAGE, bytes(NodesDiskUsage(estimates, neoStores) + estimates.SizeOfNodeProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + defensivelyPadMemoryEstimate(idMapper.MemoryEstimation(numberOfNodes))));
// A difficulty with the goal here is that we don't know how much work there is to be done in id mapper preparation stage.
// In addition to nodes themselves and SPLIT,SORT,DETECT there may be RESOLVE,SORT,DEDUPLICATE too, if there are collisions
long goal = idMapper.NeedsPreparation() ? numberOfNodes + Weighted(IdMapperPreparationStage.NAME, numberOfNodes * 4) : numberOfNodes;
InitializeProgress(goal, ImportStage.NodeImport);
}
private static long RelationshipsDiskUsage(Input_Estimates estimates, BatchingNeoStores neoStores)
{
return(estimates.NumberOfRelationships() * neoStores.RelationshipStore.RecordSize * (neoStores.UsesDoubleRelationshipRecordUnits() ? 2 : 1));
}
private static long NodesDiskUsage(Input_Estimates estimates, BatchingNeoStores neoStores)
{
return(estimates.NumberOfNodes() * neoStores.NodeStore.RecordSize + estimates.NumberOfNodeLabels());
}
private static long BaselineMemoryRequirement(BatchingNeoStores neoStores)
{
return(totalMemoryUsageOf(neoStores));
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Before public void start() throws java.io.IOException
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void Start()
{
_jobScheduler = new ThreadPoolJobScheduler();
_stores = BatchingNeoStores.batchingNeoStores(_fileSystemRule.get(), _directory.absolutePath(), Format, _config, NullLogService.Instance, AdditionalInitialIds.EMPTY, Config.defaults(), _jobScheduler);
_stores.createNew();
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public static DataStatistics importRelationships(int numRunners, org.neo4j.unsafe.impl.batchimport.input.Input input, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper idMapper, org.neo4j.unsafe.impl.batchimport.input.Collector badCollector, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, Monitor monitor, boolean validateRelationshipData) throws java.io.IOException
public static DataStatistics ImportRelationships(int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, Collector badCollector, ExecutionMonitor executionMonitor, Monitor monitor, bool validateRelationshipData)
{
DataStatistics typeDistribution = new DataStatistics(monitor, new RelationshipTypeCount[0]);
System.Func <EntityImporter> importers = () => new RelationshipImporter(stores, idMapper, typeDistribution, monitor, badCollector, validateRelationshipData, stores.UsesDoubleRelationshipRecordUnits());
ImportData(RELATIONSHIP_IMPORT_NAME, numRunners, input.Relationships(), stores, importers, executionMonitor, new MemoryUsageStatsProvider(stores, idMapper));
return(typeDistribution);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: public static void importNodes(int numRunners, org.neo4j.unsafe.impl.batchimport.input.Input input, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper idMapper, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, Monitor monitor) throws java.io.IOException
public static void ImportNodes(int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, ExecutionMonitor executionMonitor, Monitor monitor)
{
System.Func <EntityImporter> importers = () => new NodeImporter(stores, idMapper, monitor);
ImportData(NODE_IMPORT_NAME, numRunners, input.Nodes(), stores, importers, executionMonitor, new MemoryUsageStatsProvider(stores, idMapper));
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private static long importData(String title, int numRunners, InputIterable data, org.neo4j.unsafe.impl.batchimport.store.BatchingNeoStores stores, System.Func<EntityImporter> visitors, org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor executionMonitor, org.neo4j.unsafe.impl.batchimport.stats.StatsProvider memoryStatsProvider) throws java.io.IOException
private static long ImportData(string title, int numRunners, InputIterable data, BatchingNeoStores stores, System.Func <EntityImporter> visitors, ExecutionMonitor executionMonitor, StatsProvider memoryStatsProvider)
{
LongAdder roughEntityCountProgress = new LongAdder();
ExecutorService pool = Executors.newFixedThreadPool(numRunners, new NamedThreadFactory(title + "Importer"));
IoMonitor writeMonitor = new IoMonitor(stores.IoTracer);
ControllableStep step = new ControllableStep(title, roughEntityCountProgress, Configuration.DEFAULT, writeMonitor, memoryStatsProvider);
StageExecution execution = new StageExecution(title, null, Configuration.DEFAULT, Collections.singletonList(step), 0);
long startTime = currentTimeMillis();
using (InputIterator dataIterator = data.GetEnumerator())
{
executionMonitor.Start(execution);
for (int i = 0; i < numRunners; i++)
{
pool.submit(new ExhaustingEntityImporterRunnable(execution, dataIterator, visitors(), roughEntityCountProgress));
}
pool.shutdown();
long nextWait = 0;
try
{
while (!pool.awaitTermination(nextWait, TimeUnit.MILLISECONDS))
{
executionMonitor.Check(execution);
nextWait = executionMonitor.NextCheckTime() - currentTimeMillis();
}
}
catch (InterruptedException e)
{
Thread.CurrentThread.Interrupt();
throw new IOException(e);
}
}
execution.AssertHealthy();
step.MarkAsCompleted();
writeMonitor.Stop();
executionMonitor.End(execution, currentTimeMillis() - startTime);
execution.AssertHealthy();
return(roughEntityCountProgress.sum());
}
