private void InitializeRelationshipImport(Input_Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores)
{
long numberOfRelationships = estimates.NumberOfRelationships();
PrintStageHeader("(2/4) Relationship import", ESTIMATED_NUMBER_OF_RELATIONSHIPS, count(numberOfRelationships), ESTIMATED_DISK_SPACE_USAGE, bytes(RelationshipsDiskUsage(estimates, neoStores) + estimates.SizeOfRelationshipProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + totalMemoryUsageOf(idMapper)));
InitializeProgress(numberOfRelationships, ImportStage.RelationshipImport);
}
public InputAnonymousInnerClass(InputIterable nodes, InputIterable relationships, IdMapper idMapper, [email protected] badCollector, Input_Estimates estimates) { this._nodes = nodes; this._relationships = relationships; this._idMapper = idMapper; this._badCollector = badCollector; this._estimates = estimates; }
private void InitializeNodeImport(Input_Estimates estimates, IdMapper idMapper, BatchingNeoStores neoStores)
{
long numberOfNodes = estimates.NumberOfNodes();
PrintStageHeader("(1/4) Node import", ESTIMATED_NUMBER_OF_NODES, count(numberOfNodes), ESTIMATED_DISK_SPACE_USAGE, bytes(NodesDiskUsage(estimates, neoStores) + estimates.SizeOfNodeProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(BaselineMemoryRequirement(neoStores) + defensivelyPadMemoryEstimate(idMapper.MemoryEstimation(numberOfNodes))));
// A difficulty with the goal here is that we don't know how much work there is to be done in id mapper preparation stage.
// In addition to nodes themselves and SPLIT,SORT,DETECT there may be RESOLVE,SORT,DEDUPLICATE too, if there are collisions
long goal = idMapper.NeedsPreparation() ? numberOfNodes + Weighted(IdMapperPreparationStage.NAME, numberOfNodes * 4) : numberOfNodes;
InitializeProgress(goal, ImportStage.NodeImport);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldNotWantIfCountWayLowerThanCapacity()
public virtual void ShouldNotWantIfCountWayLowerThanCapacity()
{
// given
RecordFormats formats = Standard.LATEST_RECORD_FORMATS;
Monitor monitor = mock(typeof(Monitor));
Input_Estimates estimates = Inputs.KnownEstimates(1000, 1000, 0, 0, 0, 0, 0); // we don't care about the rest of the estimates in this checking
// when
(new EstimationSanityChecker(formats, monitor)).SanityCheck(estimates);
// then
verifyNoMoreInteractions(monitor);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldWarnAboutCounthigherThanCapacity()
public virtual void ShouldWarnAboutCounthigherThanCapacity()
{
// given
RecordFormats formats = Standard.LATEST_RECORD_FORMATS;
Monitor monitor = mock(typeof(Monitor));
Input_Estimates estimates = Inputs.KnownEstimates(formats.Node().MaxId * 2, formats.Relationship().MaxId * 2, 0, 0, 0, 0, 0); // we don't care about the rest of the estimates in this checking
// when
(new EstimationSanityChecker(formats, monitor)).SanityCheck(estimates);
// then
verify(monitor).mayExceedNodeIdCapacity(formats.Node().MaxId, estimates.NumberOfNodes());
verify(monitor).mayExceedRelationshipIdCapacity(formats.Relationship().MaxId, estimates.NumberOfRelationships());
}
public override void Initialize(DependencyResolver dependencyResolver)
{
this._dependencyResolver = dependencyResolver;
Input_Estimates estimates = dependencyResolver.ResolveDependency(typeof(Input_Estimates));
BatchingNeoStores neoStores = dependencyResolver.ResolveDependency(typeof(BatchingNeoStores));
IdMapper idMapper = dependencyResolver.ResolveDependency(typeof(IdMapper));
NodeRelationshipCache nodeRelationshipCache = dependencyResolver.ResolveDependency(typeof(NodeRelationshipCache));
_pageCacheArrayFactoryMonitor = dependencyResolver.ResolveDependency(typeof(PageCacheArrayFactoryMonitor));
long biggestCacheMemory = estimatedCacheSize(neoStores, nodeRelationshipCache.MemoryEstimation(estimates.NumberOfNodes()), idMapper.MemoryEstimation(estimates.NumberOfNodes()));
PrintStageHeader("Import starting", ESTIMATED_NUMBER_OF_NODES, count(estimates.NumberOfNodes()), ESTIMATED_NUMBER_OF_NODE_PROPERTIES, count(estimates.NumberOfNodeProperties()), ESTIMATED_NUMBER_OF_RELATIONSHIPS, count(estimates.NumberOfRelationships()), ESTIMATED_NUMBER_OF_RELATIONSHIP_PROPERTIES, count(estimates.NumberOfRelationshipProperties()), ESTIMATED_DISK_SPACE_USAGE, bytes(NodesDiskUsage(estimates, neoStores) + RelationshipsDiskUsage(estimates, neoStores) + estimates.SizeOfNodeProperties() + estimates.SizeOfRelationshipProperties()), ESTIMATED_REQUIRED_MEMORY_USAGE, bytes(biggestCacheMemory));
Console.WriteLine();
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void migrateWithBatchImporter(org.neo4j.io.layout.DatabaseLayout sourceDirectoryStructure, org.neo4j.io.layout.DatabaseLayout migrationDirectoryStructure, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, org.neo4j.kernel.impl.util.monitoring.ProgressReporter progressReporter, org.neo4j.kernel.impl.store.format.RecordFormats oldFormat, org.neo4j.kernel.impl.store.format.RecordFormats newFormat) throws java.io.IOException
private void MigrateWithBatchImporter(DatabaseLayout sourceDirectoryStructure, DatabaseLayout migrationDirectoryStructure, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, ProgressReporter progressReporter, RecordFormats oldFormat, RecordFormats newFormat)
{
PrepareBatchImportMigration(sourceDirectoryStructure, migrationDirectoryStructure, oldFormat, newFormat);
bool requiresDynamicStoreMigration = !newFormat.Dynamic().Equals(oldFormat.Dynamic());
bool requiresPropertyMigration = !newFormat.Property().Equals(oldFormat.Property()) || requiresDynamicStoreMigration;
File badFile = sourceDirectoryStructure.File([email protected]_Fields.BAD_FILE_NAME);
using (NeoStores legacyStore = InstantiateLegacyStore(oldFormat, sourceDirectoryStructure), Stream badOutput = new BufferedOutputStream(new FileStream(badFile, false)))
{
Configuration importConfig = new Configuration_OverriddenAnonymousInnerClass(this, _config, sourceDirectoryStructure);
AdditionalInitialIds additionalInitialIds = ReadAdditionalIds(lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset);
// We have to make sure to keep the token ids if we're migrating properties/labels
BatchImporter importer = BatchImporterFactory.withHighestPriority().instantiate(migrationDirectoryStructure, _fileSystem, _pageCache, importConfig, _logService, withDynamicProcessorAssignment(MigrationBatchImporterMonitor(legacyStore, progressReporter, importConfig), importConfig), additionalInitialIds, _config, newFormat, NO_MONITOR, _jobScheduler);
InputIterable nodes = () => LegacyNodesAsInput(legacyStore, requiresPropertyMigration);
InputIterable relationships = () => LegacyRelationshipsAsInput(legacyStore, requiresPropertyMigration);
long propertyStoreSize = StoreSize(legacyStore.PropertyStore) / 2 + StoreSize(legacyStore.PropertyStore.StringStore) / 2 + StoreSize(legacyStore.PropertyStore.ArrayStore) / 2;
Input_Estimates estimates = knownEstimates(legacyStore.NodeStore.NumberOfIdsInUse, legacyStore.RelationshipStore.NumberOfIdsInUse, legacyStore.PropertyStore.NumberOfIdsInUse, legacyStore.PropertyStore.NumberOfIdsInUse, propertyStoreSize / 2, propertyStoreSize / 2, 0);
importer.DoImport(Inputs.input(nodes, relationships, IdMappers.actual(), Collectors.badCollector(badOutput, 0), estimates));
// During migration the batch importer doesn't necessarily writes all entities, depending on
// which stores needs migration. Node, relationship, relationship group stores are always written
// anyways and cannot be avoided with the importer, but delete the store files that weren't written
// (left empty) so that we don't overwrite those in the real store directory later.
ICollection <DatabaseFile> storesToDeleteFromMigratedDirectory = new List <DatabaseFile>();
storesToDeleteFromMigratedDirectory.Add(DatabaseFile.METADATA_STORE);
if (!requiresPropertyMigration)
{
// We didn't migrate properties, so the property stores in the migrated store are just empty/bogus
storesToDeleteFromMigratedDirectory.addAll(asList(DatabaseFile.PROPERTY_STORE, DatabaseFile.PROPERTY_STRING_STORE, DatabaseFile.PROPERTY_ARRAY_STORE));
}
if (!requiresDynamicStoreMigration)
{
// We didn't migrate labels (dynamic node labels) or any other dynamic store
storesToDeleteFromMigratedDirectory.addAll(asList(DatabaseFile.NODE_LABEL_STORE, DatabaseFile.LABEL_TOKEN_STORE, DatabaseFile.LABEL_TOKEN_NAMES_STORE, DatabaseFile.RELATIONSHIP_TYPE_TOKEN_STORE, DatabaseFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, DatabaseFile.PROPERTY_KEY_TOKEN_STORE, DatabaseFile.PROPERTY_KEY_TOKEN_NAMES_STORE, DatabaseFile.SCHEMA_STORE));
}
fileOperation(DELETE, _fileSystem, migrationDirectoryStructure, migrationDirectoryStructure, storesToDeleteFromMigratedDirectory, true, null);
}
}
private static long RelationshipsDiskUsage(Input_Estimates estimates, BatchingNeoStores neoStores)
{
return(estimates.NumberOfRelationships() * neoStores.RelationshipStore.RecordSize * (neoStores.UsesDoubleRelationshipRecordUnits() ? 2 : 1));
}
private static long NodesDiskUsage(Input_Estimates estimates, BatchingNeoStores neoStores)
{
return(estimates.NumberOfNodes() * neoStores.NodeStore.RecordSize + estimates.NumberOfNodeLabels());
}
public virtual void SanityCheck(Input_Estimates estimates)
{
SanityCheckEstimateWithMaxId(estimates.NumberOfNodes(), _formats.node().MaxId, _monitor.mayExceedNodeIdCapacity);
SanityCheckEstimateWithMaxId(estimates.NumberOfRelationships(), _formats.relationship().MaxId, _monitor.mayExceedRelationshipIdCapacity);
}
//JAVA TO C# CONVERTER WARNING: 'final' parameters are ignored unless the option to convert to C# 7.2 'in' parameters is selected:
//ORIGINAL LINE: public static Input input(final org.neo4j.unsafe.impl.batchimport.InputIterable nodes, final org.neo4j.unsafe.impl.batchimport.InputIterable relationships, final org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper idMapper, final Collector badCollector, org.neo4j.unsafe.impl.batchimport.input.Input_Estimates estimates)
public static Input Input(InputIterable nodes, InputIterable relationships, IdMapper idMapper, Collector badCollector, Input_Estimates estimates)
{
return(new InputAnonymousInnerClass(nodes, relationships, idMapper, badCollector, estimates));
}