/// <summary>
/// There was this problem where some steps and in particular parallel CSV input parsing that
/// paniced would hang the import entirely.
/// </summary>
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldExitAndThrowExceptionOnPanic() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldExitAndThrowExceptionOnPanic()
{
try
{
using (JobScheduler jobScheduler = new ThreadPoolJobScheduler())
{
BatchImporter importer = new ParallelBatchImporter(_directory.databaseLayout(), _fs, null, Configuration.DEFAULT, NullLogService.Instance, ExecutionMonitors.invisible(), AdditionalInitialIds.EMPTY, Config.defaults(), StandardV3_0.RECORD_FORMATS, NO_MONITOR, jobScheduler);
IEnumerable <DataFactory> nodeData = datas(data(NO_DECORATOR, FileAsCharReadable(NodeCsvFileWithBrokenEntries())));
Input brokenCsvInput = new CsvInput(nodeData, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, CsvConfigurationWithLowBufferSize(), new BadCollector(NullOutputStream.NULL_OUTPUT_STREAM, 0, 0), CsvInput.NO_MONITOR);
importer.DoImport(brokenCsvInput);
fail("Should have failed properly");
}
}
catch (InputException e)
{
// THEN
assertTrue(e.InnerException is DataAfterQuoteException);
// and we managed to shut down properly
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldCalculateCorrectEstimates() throws Exception
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldCalculateCorrectEstimates()
{
// given a couple of input files of various layouts
Input input = GenerateData();
RecordFormats format = LATEST_RECORD_FORMATS;
[email protected]_Estimates estimates = input.CalculateEstimates(new PropertyValueRecordSizeCalculator(format.Property().getRecordSize(NO_STORE_HEADER), parseInt(GraphDatabaseSettings.string_block_size.DefaultValue), 0, parseInt(GraphDatabaseSettings.array_block_size.DefaultValue), 0));
// when
DatabaseLayout databaseLayout = Directory.databaseLayout();
Config config = Config.defaults();
FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
using (JobScheduler jobScheduler = new ThreadPoolJobScheduler())
{
(new ParallelBatchImporter(databaseLayout, fs, null, Configuration.DEFAULT, NullLogService.Instance, ExecutionMonitors.invisible(), AdditionalInitialIds.EMPTY, config, format, NO_MONITOR, jobScheduler)).doImport(input);
// then compare estimates with actual disk sizes
VersionContextSupplier contextSupplier = EmptyVersionContextSupplier.EMPTY;
using (PageCache pageCache = (new ConfiguringPageCacheFactory(fs, config, PageCacheTracer.NULL, Org.Neo4j.Io.pagecache.tracing.cursor.PageCursorTracerSupplier_Fields.Null, NullLog.Instance, contextSupplier, jobScheduler)).OrCreatePageCache, NeoStores stores = (new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs), pageCache, fs, NullLogProvider.Instance, contextSupplier)).openAllNeoStores())
{
AssertRoughlyEqual(estimates.NumberOfNodes(), stores.NodeStore.NumberOfIdsInUse);
AssertRoughlyEqual(estimates.NumberOfRelationships(), stores.RelationshipStore.NumberOfIdsInUse);
AssertRoughlyEqual(estimates.NumberOfNodeProperties() + estimates.NumberOfRelationshipProperties(), CalculateNumberOfProperties(stores));
}
AssertRoughlyEqual(PropertyStorageSize(), estimates.SizeOfNodeProperties() + estimates.SizeOfRelationshipProperties());
}
}
private void Defrag(int nodeCount, RecordStore <RelationshipGroupRecord> groupStore)
{
Monitor monitor = mock(typeof(Monitor));
RelationshipGroupDefragmenter defragmenter = new RelationshipGroupDefragmenter(_config, ExecutionMonitors.invisible(), monitor, AUTO_WITHOUT_PAGECACHE);
// Calculation below correlates somewhat to calculation in RelationshipGroupDefragmenter.
// Anyway we verify below that we exercise the multi-pass bit, which is what we want
long memory = groupStore.HighId * 15 + 200;
defragmenter.Run(memory, _stores, nodeCount);
// Verify that we exercise the multi-pass functionality
verify(monitor, atLeast(2)).defragmentingNodeRange(anyLong(), anyLong());
verify(monitor, atMost(10)).defragmentingNodeRange(anyLong(), anyLong());
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void shouldBehaveCorrectlyUnderStress() throws Throwable
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
public virtual void ShouldBehaveCorrectlyUnderStress()
{
long durationInMinutes = parseLong(fromEnv("CHECK_POINT_LOG_ROTATION_STRESS_DURATION", DEFAULT_DURATION_IN_MINUTES));
File storeDir = new File(fromEnv("CHECK_POINT_LOG_ROTATION_STORE_DIRECTORY", _defaultStoreDir));
long nodeCount = parseLong(fromEnv("CHECK_POINT_LOG_ROTATION_NODE_COUNT", DEFAULT_NODE_COUNT));
int threads = parseInt(fromEnv("CHECK_POINT_LOG_ROTATION_WORKER_THREADS", DEFAULT_WORKER_THREADS));
string pageCacheMemory = fromEnv("CHECK_POINT_LOG_ROTATION_PAGE_CACHE_MEMORY", DEFAULT_PAGE_CACHE_MEMORY);
Console.WriteLine("1/6\tBuilding initial store...");
using (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(), JobScheduler jobScheduler = new ThreadPoolJobScheduler())
{
Config dbConfig = Config.defaults();
(new ParallelBatchImporter(DatabaseLayout.of(ensureExistsAndEmpty(storeDir)), fileSystem, null, DEFAULT, NullLogService.Instance, ExecutionMonitors.defaultVisible(jobScheduler), EMPTY, dbConfig, RecordFormatSelector.selectForConfig(dbConfig, NullLogProvider.Instance), NO_MONITOR, jobScheduler)).doImport(new NodeCountInputs(nodeCount));
}
Console.WriteLine("2/6\tStarting database...");
GraphDatabaseBuilder builder = (new TestGraphDatabaseFactory()).newEmbeddedDatabaseBuilder(storeDir);
GraphDatabaseService db = builder.SetConfig(GraphDatabaseSettings.pagecache_memory, pageCacheMemory).setConfig(GraphDatabaseSettings.keep_logical_logs, Settings.FALSE).setConfig(GraphDatabaseSettings.check_point_interval_time, CHECK_POINT_INTERVAL_MINUTES + "m").setConfig(GraphDatabaseSettings.tracer, "timer").newGraphDatabase();
Console.WriteLine("3/6\tWarm up db...");
using (Workload workload = new Workload(db, defaultRandomMutation(nodeCount, db), threads))
{
// make sure to run at least one checkpoint during warmup
long warmUpTimeMillis = TimeUnit.SECONDS.toMillis(CHECK_POINT_INTERVAL_MINUTES * 2);
workload.Run(warmUpTimeMillis, Workload.TransactionThroughput_Fields.NONE);
}
Console.WriteLine("4/6\tStarting workload...");
TransactionThroughputChecker throughput = new TransactionThroughputChecker();
using (Workload workload = new Workload(db, defaultRandomMutation(nodeCount, db), threads))
{
workload.Run(TimeUnit.MINUTES.toMillis(durationInMinutes), throughput);
}
Console.WriteLine("5/6\tShutting down...");
Db.shutdown();
try
{
Console.WriteLine("6/6\tPrinting stats and recorded timings...");
TimerTransactionTracer.printStats(System.out);
throughput.AssertThroughput(System.out);
}
finally
{
Console.WriteLine("Done.");
}
// let's cleanup disk space when everything went well
FileUtils.deleteRecursively(storeDir);
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#:
//ORIGINAL LINE: private void createRandomData(int count) throws Exception
private void CreateRandomData(int count)
{
Config config = Config.defaults();
RecordFormats recordFormats = RecordFormatSelector.selectForConfig(config, NullLogProvider.Instance);
using (RandomDataInput input = new RandomDataInput(this, count), JobScheduler jobScheduler = new ThreadPoolJobScheduler())
{
BatchImporter importer = new ParallelBatchImporter(_directory.databaseLayout(), _fileSystemRule.get(), null, DEFAULT, NullLogService.Instance, ExecutionMonitors.invisible(), EMPTY, config, recordFormats, NO_MONITOR, jobScheduler);
importer.DoImport(input);
}
}
