/// <summary> /// Base size of blocks of entries. As entries gets written to a BlockStorage, they are buffered up to this size, then sorted and written out. /// As blocks gets merged into bigger blocks, this is still the size of the read buffer for each block no matter its size. /// Each thread has its own buffer when writing and each thread has <seealso cref="mergeFactor"/> buffers when merging. /// The memory usage will be at its biggest during merge and a total memory usage sum can be calculated like so: /// /// blockSize * numberOfPopulationWorkers * <seealso cref="mergeFactor"/> /// /// where typically <seealso cref="BatchingMultipleIndexPopulator"/> controls the number of population workers. The setting /// `unsupported.dbms.multi_threaded_schema_index_population_enabled` controls whether or not the multi-threaded <seealso cref="BatchingMultipleIndexPopulator"/> /// is used, otherwise a single-threaded populator is used instead. /// </summary> public static int ParseBlockSize() { long blockSize = ByteUnit.parse(FeatureToggles.getString(typeof(BlockBasedIndexPopulator), BLOCK_SIZE_NAME, "1M")); Preconditions.checkArgument(blockSize >= 20 && blockSize < int.MaxValue, "Block size need to fit in int. Was " + blockSize); return(( int )blockSize); }
/// <summary> /// Devuelve la representación de una cantidad de bytes en la unidad más alta posible. /// Se utilizan las clases derivadas de Models.SizeUnit para esto. /// </summary> private string ParseSize(int value) { var b = new ByteUnit(); b.Value = value; return(b.Get()); }
/// <summary> /// 根据字节单位获取对应名称 /// </summary> /// <param name="unit">单位</param> /// <returns></returns> public string GetUnitName(ByteUnit unit) { switch (unit) { case ByteUnit.B: { return("B"); } case ByteUnit.GB: { return("GB"); } case ByteUnit.KB: { return("KB"); } case ByteUnit.MB: { return("MB"); } default: { throw new Exception("未知单位!"); } } }
public override void setValueWithOldSetting(string value, IDictionary <string, string> rawConfiguration) { if (StringUtils.isNotEmpty(value)) { string oldSettingDefaultValue = GraphDatabaseSettings.index_sampling_buffer_size.DefaultValue; long? newValue = oldSettingDefaultValue.Equals(value) ? ByteUnit.mebiBytes(8) : Settings.BYTES.apply(value); rawConfiguration["dbms.index_sampling.sample_size_limit"] = newValue.ToString(); } }
/// <summary> /// Converts a byte value into the requested <see cref="ByteUnit"/>. /// </summary> /// <param name="bytes">The byte value to convert.</param> /// <param name="unit">The <see cref="ByteUnit"/> to convert into.</param> /// <returns> /// Returns a double value in the <see cref="ByteUnit"/> requested by the caller. /// </returns> /// <exception cref="ArgumentOutOfRangeException"> /// Exception will be thrown if the <see cref="ByteUnit"/> enumeration /// was out of the defined range. /// </exception> public static double ConvertBytes(double bytes, ByteUnit unit) { if (!Enum.IsDefined(typeof(ByteUnit), unit)) { throw new ArgumentOutOfRangeException("unit", unit, "Undefined value for ByteUnit."); } double result = bytes / System.Math.Pow(1024, Convert.ToDouble((int)unit)); return result; }
/// <summary> /// Converts a byte value into the requested <see cref="ByteUnit"/>. /// </summary> /// <param name="bytes">The byte value to convert.</param> /// <param name="unit">The <see cref="ByteUnit"/> to convert into.</param> /// <returns> /// Returns a double value in the <see cref="ByteUnit"/> requested by the caller. /// </returns> /// <exception cref="ArgumentOutOfRangeException"> /// Exception will be thrown if the <see cref="ByteUnit"/> enumeration /// was out of the defined range. /// </exception> public static double ConvertBytes(double bytes, ByteUnit unit) { if (!Enum.IsDefined(typeof(ByteUnit), unit)) { throw new ArgumentOutOfRangeException("unit", unit, "Undefined value for ByteUnit."); } double result = bytes / System.Math.Pow(1024, Convert.ToDouble((int)unit)); return(result); }
public static void ConvToNearsetUnit(long num, ByteUnit unit, out long newNum, out ByteUnit newUnit) { long n = num; ByteUnit u = unit; while (n >= 1024 && u + 1 < ByteUnit.Limit) { n /= 1024; u += 1; } newNum = n; newUnit = u; }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test void trackMemoryAllocations() internal virtual void TrackMemoryAllocations() { LocalMemoryTracker memoryTracker = new LocalMemoryTracker(); GrabAllocator allocator = ( GrabAllocator )MemoryAllocator.createAllocator("2m", memoryTracker); assertEquals(0, memoryTracker.UsedDirectMemory()); long pointer = allocator.AllocateAligned(ByteUnit.mebiBytes(1), 1); assertEquals(ByteUnit.mebiBytes(1), memoryTracker.UsedDirectMemory()); allocator.Close(); assertEquals(0, memoryTracker.UsedDirectMemory()); }
public static decimal GetFactor(this ByteUnit unit) { var power = unit.GetInfo().Power; var factor = 1M; if (power > 0) { for (var i = 0; i < power; i++) { factor *= 1024; } } return(factor); }
public static long DefaultHeuristicPageCacheMemory() { // First check if we have a default override... string defaultMemoryOverride = System.getProperty("dbms.pagecache.memory.default.override"); if (!string.ReferenceEquals(defaultMemoryOverride, null)) { return(BYTES.apply(defaultMemoryOverride)); } double ratioOfFreeMem = 0.50; string defaultMemoryRatioOverride = System.getProperty("dbms.pagecache.memory.ratio.default.override"); if (!string.ReferenceEquals(defaultMemoryRatioOverride, null)) { ratioOfFreeMem = double.Parse(defaultMemoryRatioOverride); } // Try to compute (RAM - maxheap) * 0.50 if we can get reliable numbers... long maxHeapMemory = Runtime.Runtime.maxMemory(); if (0 < maxHeapMemory && maxHeapMemory < long.MaxValue) { try { long physicalMemory = OsBeanUtil.TotalPhysicalMemory; if (0 < physicalMemory && physicalMemory < long.MaxValue && maxHeapMemory < physicalMemory) { long heuristic = ( long )((physicalMemory - maxHeapMemory) * ratioOfFreeMem); long min = ByteUnit.mebiBytes(32); // We'd like at least 32 MiBs. long max = Math.Min(maxHeapMemory * 70, ByteUnit.gibiBytes(20)); // Don't heuristically take more than 20 GiBs, and don't take more than 70 times our max heap. // 20 GiBs of page cache memory is ~2.6 million 8 KiB pages. If each page has an overhead of // 72 bytes, then this will take up ~175 MiBs of heap memory. We should be able to tolerate that // in most environments. The "no more than 70 times heap" heuristic is based on the page size over // the per page overhead, 8192 / 72 ~= 114, plus leaving some extra room on the heap for the rest // of the system. This means that we won't heuristically try to create a page cache that is too // large to fit on the heap. return(Math.Min(max, Math.Max(min, heuristic))); } } catch (Exception) { } } // ... otherwise we just go with 2 GiBs. return(ByteUnit.gibiBytes(2)); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void reheatingMustWorkOnLargeNumberOfPages() throws Exception //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ReheatingMustWorkOnLargeNumberOfPages() { int maxPagesInMemory = 1_000; int[] pageIds = RandomSortedPageIds(maxPagesInMemory); string pageCacheMemory = (maxPagesInMemory * ByteUnit.kibiBytes(9)).ToString(); using (PageCache pageCache = _pageCacheRule.getPageCache(_fs, _cfg.withMemory(pageCacheMemory)), PagedFile pf = pageCache.Map(_file, pageCache.PageSize(), StandardOpenOption.CREATE)) { using (PageCursor writer = pf.Io(0, Org.Neo4j.Io.pagecache.PagedFile_Fields.PfSharedWriteLock)) { foreach (int pageId in pageIds) { assertTrue(writer.Next(pageId)); } } pf.FlushAndForce(); PageCacheWarmer warmer = new PageCacheWarmer(_fs, pageCache, _scheduler, _testDirectory.databaseDir()); warmer.Profile(); } long initialFaults = _cacheTracer.faults(); ClearTracerCounts(); using (PageCache pageCache = _pageCacheRule.getPageCache(_fs, _cfg), PagedFile pf = pageCache.Map(_file, pageCache.PageSize())) { PageCacheWarmer warmer = new PageCacheWarmer(_fs, pageCache, _scheduler, _testDirectory.databaseDir()); warmer.Start(); warmer.Reheat(); pageCache.ReportEvents(); assertThat(_cacheTracer.faults(), @is(initialFaults + pageIds.Length)); using (PageCursor reader = pf.Io(0, Org.Neo4j.Io.pagecache.PagedFile_Fields.PF_SHARED_READ_LOCK)) { foreach (int pageId in pageIds) { assertTrue(reader.Next(pageId)); } } // No additional faults must have been reported. pageCache.ReportEvents(); assertThat(_cacheTracer.faults(), @is(initialFaults + pageIds.Length)); } }
private static decimal ToSize(long bytes, ByteUnit inUnit, int decimalPlaces = DEFAULT_PLACES) { switch (inUnit) { case ByteUnit.MB: return(Calculate(bytes, Megabytes, decimalPlaces)); case ByteUnit.KB: return(Calculate(bytes, Kilobytes, decimalPlaces)); case ByteUnit.TB: return(Calculate(bytes, Terabytes, decimalPlaces)); default: return(Calculate(bytes, Gigabytes, decimalPlaces)); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: private static void transactions1M(org.neo4j.causalclustering.discovery.Cluster<?> cluster) throws Exception private static void Transactions1M <T1>(Cluster <T1> cluster) { int numberOfTransactions = 500; long sizeOfTransaction = (ByteUnit.mebiBytes(1) / numberOfTransactions) + 1; for (int txId = 0; txId < numberOfTransactions; txId++) { cluster.CoreTx((coreGraphDatabase, transaction) => { Node node = coreGraphDatabase.createNode(); string longString = LongStream.range(0, sizeOfTransaction).map(l => l % 10).mapToObj(long?.toString).collect(joining("")); node.setProperty("name", longString); coreGraphDatabase.createNode().createRelationshipTo(node, RelationshipType.withName("KNOWS")); transaction.success(); }); } }
private static void Transactions1M(GraphDatabaseService db) { int numberOfTransactions = 500; long sizeOfTransaction = (ByteUnit.mebiBytes(1) / numberOfTransactions) + 1; for (int txId = 0; txId < numberOfTransactions; txId++) { using (Transaction tx = Db.beginTx()) { Node node = Db.createNode(); string longString = LongStream.range(0, sizeOfTransaction).map(l => l % 10).mapToObj(long?.toString).collect(joining("")); node.SetProperty("name", longString); Db.createNode().createRelationshipTo(node, RelationshipType.withName("KNOWS")); tx.Success(); } } }
public virtual decimal ToSize(ByteUnit inUnit, int decimalPlaces) { switch (inUnit) { case ByteUnit.MB: return(Calculate(this.SizeOnDisk, MB, decimalPlaces)); case ByteUnit.KB: return(Calculate(this.SizeOnDisk, KB, decimalPlaces)); case ByteUnit.TB: return(Calculate(this.SizeOnDisk, TB, decimalPlaces)); default: return(Calculate(this.SizeOnDisk, GB, decimalPlaces)); } }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test void allowStoreThatExceedDefaultSize() throws java.io.IOException //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: internal virtual void AllowStoreThatExceedDefaultSize() { File aFile = new File("test"); StoreChannel channel = _fs.open(aFile, OpenMode.READ_WRITE); ByteBuffer buffer = allocate(Long.BYTES); int mebiBytes = ( int )ByteUnit.mebiBytes(1); for (int position = mebiBytes + 42; position < 10_000_000; position += mebiBytes) { buffer.putLong(1); buffer.flip(); channel.WriteAll(buffer, position); buffer.clear(); } channel.close(); }
public static double ConvertByteValue(double byteVal, ByteUnit byteUnit, int decimalPrecision = 2, ByteUnit startingUnit = ByteUnit.Byte) { if (byteUnit == startingUnit || byteVal == 0) { return(Math.Round(byteVal, decimalPrecision)); } int byteUnitInt = (int)byteUnit; int startingUnitInt = (int)startingUnit; double newVal = byteVal; while (startingUnitInt < byteUnitInt) { newVal /= 1024; startingUnitInt++; } return(Math.Round(newVal, decimalPrecision)); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test void growEphemeralFileBuffer() internal virtual void GrowEphemeralFileBuffer() { EphemeralFileSystemAbstraction.DynamicByteBuffer byteBuffer = new EphemeralFileSystemAbstraction.DynamicByteBuffer(); sbyte[] testBytes = new sbyte[] { 1, 2, 3, 4 }; int length = testBytes.Length; byteBuffer.Put(0, testBytes, 0, length); assertEquals(( int )ByteUnit.kibiBytes(1), byteBuffer.Buf().capacity()); byteBuffer.Put(( int )(ByteUnit.kibiBytes(1) + 2), testBytes, 0, length); assertEquals(( int )ByteUnit.kibiBytes(2), byteBuffer.Buf().capacity()); byteBuffer.Put(( int )(ByteUnit.kibiBytes(5) + 2), testBytes, 0, length); assertEquals(( int )ByteUnit.kibiBytes(8), byteBuffer.Buf().capacity()); byteBuffer.Put(( int )(ByteUnit.mebiBytes(2) + 2), testBytes, 0, length); assertEquals(( int )ByteUnit.mebiBytes(4), byteBuffer.Buf().capacity()); }
/// <summary> /// 根据字节数获取字节单位 /// </summary> /// <param name="byteCount">字节数</param> /// <returns></returns> public ByteUnit ConvertToUnitFromByteCount(long byteCount) { ByteUnit size = ByteUnit.B; if (byteCount >= 1073741824) { size = ByteUnit.GB; } else if (byteCount >= 1048576) { size = ByteUnit.MB; } else if (byteCount >= 1024) { size = ByteUnit.KB; } else if (byteCount > 0 && byteCount < 1024) { size = ByteUnit.B; } return(size); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test void veryLargePageListsMustBeFullyAccessible() internal virtual void VeryLargePageListsMustBeFullyAccessible() { // We need roughly 2 GiBs of memory for the meta-data here, which is why this is an IT and not a Test. // We add one extra page worth of data to the size here, to avoid ending up on a "convenient" boundary. int pageSize = ( int )ByteUnit.kibiBytes(8); long pageCacheSize = ByteUnit.gibiBytes(513) + pageSize; int pages = Math.toIntExact(pageCacheSize / pageSize); MemoryAllocator mman = MemoryAllocator.createAllocator("2 GiB", GlobalMemoryTracker.INSTANCE); SwapperSet swappers = new SwapperSet(); long victimPage = VictimPageReference.GetVictimPage(pageSize, GlobalMemoryTracker.INSTANCE); PageList pageList = new PageList(pages, pageSize, mman, swappers, victimPage, Long.BYTES); // Verify we end up with the correct number of pages. assertThat(pageList.PageCount, @is(pages)); // Spot-check the accessibility in the bulk of the pages. IntStream.range(0, pages / 32).parallel().forEach(id => verifyPageMetaDataIsAccessible(pageList, id * 32)); // Thoroughly check the accessibility around the tail end of the page list. IntStream.range(pages - 2000, pages).parallel().forEach(id => verifyPageMetaDataIsAccessible(pageList, id)); }
public static UnitAttribute GetInfo(this ByteUnit unit) { return(Enum <ByteUnit> .GetAttribute <UnitAttribute>(unit)); }
public static string AutoToSize(this Int32 val) { ByteUnit unit = (ByteUnit)Math.Log(val, 1000); return(val.ToSize(unit) + " " + unit); }
/// <summary> /// Overloaded method that gets the free space left on the disk given by a directory. /// </summary> /// <param name="directory">Directory where the free disk space should be counted.</param> /// <param name="unit">The <see cref="ByteUnit"/> in which the result should be returned.</param> /// <returns> /// Returns the free disk space in the requested unit, i.e. MegaBytes / GigaBytes and so on. /// </returns> public static double GetDiskFreeSpace(string directory, ByteUnit unit) { double diskFreeBytes = GetDiskFreeSpace(directory); return ConvertBytes(diskFreeBytes, unit); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @BeforeEach public void setUp() public virtual void SetUp() { _tracer = new DefaultPageCacheTracer(); _swapper = new DummyPageSwapper("filename", ( int )ByteUnit.kibiBytes(8)); }
public static string ToSize(this Int32 val, ByteUnit unit) { return((val / Math.Pow(1024, (Int64)unit)).ToString("0.00")); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void shouldKeepTransactionsIntactWhenConcurrentlyRotationAndAppending() throws Throwable //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: public virtual void ShouldKeepTransactionsIntactWhenConcurrentlyRotationAndAppending() { // GIVEN LogVersionRepository logVersionRepository = new SimpleLogVersionRepository(); LogFiles logFiles = LogFilesBuilder.builder(_directory.databaseLayout(), _fileSystemRule.get()).withLogVersionRepository(logVersionRepository).withRotationThreshold(ByteUnit.mebiBytes(1)).withTransactionIdStore(new SimpleTransactionIdStore()).build(); _life.add(logFiles); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final java.util.concurrent.atomic.AtomicBoolean end = new java.util.concurrent.atomic.AtomicBoolean(); AtomicBoolean end = new AtomicBoolean(); AllTheMonitoring monitoring = new AllTheMonitoring(end, 100); TransactionIdStore txIdStore = new SimpleTransactionIdStore(); TransactionMetadataCache metadataCache = new TransactionMetadataCache(); monitoring.LogFile = logFiles.LogFile; DatabaseHealth health = new DatabaseHealth(mock(typeof(DatabasePanicEventGenerator)), NullLog.Instance); LogRotation rotation = new LogRotationImpl(monitoring, logFiles, health); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFiles, rotation, metadataCache, txIdStore, BYPASS, health)); TransactionAppender appender = _life.add(new BatchingTransactionAppender(logFiles, rotation, metadataCache, txIdStore, BYPASS, health)); // WHEN Race race = new Race(); for (int i = 0; i < 10; i++) { race.AddContestant(() => { while (!end.get()) { try { appender.Append(new TransactionToApply(SillyTransaction(1_000)), NULL); } catch (Exception e) { e.printStackTrace(System.out); end.set(true); fail(e.Message); } } }); } race.AddContestant(EndAfterMax(10, SECONDS, end)); race.Go(); // THEN assertTrue(monitoring.NumberOfRotations() > 0); }
/// <summary> /// Overloaded method that gets the free space left on the disk given by a directory. /// </summary> /// <param name="directory">Directory where the free disk space should be counted.</param> /// <param name="unit">The <see cref="ByteUnit"/> in which the result should be returned.</param> /// <returns> /// Returns the free disk space in the requested unit, i.e. MegaBytes / GigaBytes and so on. /// </returns> public static double GetDiskFreeSpace(string directory, ByteUnit unit) { double diskFreeBytes = GetDiskFreeSpace(directory); return(ConvertBytes(diskFreeBytes, unit)); }
public FixedLinkedStubPageCursorAnonymousInnerClass() : base(0, (int)ByteUnit.kibiBytes(4)) { }
public static SuffixesAttribute GetSuffixes(this ByteUnit unit) { return(Enum <ByteUnit> .GetAttribute <SuffixesAttribute>(unit)); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @BeforeEach void setUp() internal virtual void SetUp() { _cacheTracer = new DefaultPageCacheTracer(); _pageCursorTracer = CreateTracer(); _swapper = new DummyPageSwapper("filename", ( int )ByteUnit.kibiBytes(8)); }
/// <summary> /// Trigger store flush (checkpoint) and write <seealso cref="NeoStoreDataSource.listStoreFiles(bool) store files"/> to the /// given <seealso cref="StoreWriter"/>. /// </summary> /// <param name="triggerName"> name of the component asks for store files. </param> /// <param name="writer"> store writer to write files to. </param> /// <param name="includeLogs"> <code>true</code> if transaction logs should be copied, <code>false</code> otherwise. </param> /// <returns> a <seealso cref="RequestContext"/> specifying at which point the store copy started. </returns> public virtual RequestContext FlushStoresAndStreamStoreFiles(string triggerName, StoreWriter writer, bool includeLogs) { try { string storeCopyIdentifier = Thread.CurrentThread.Name; ThrowingAction <IOException> checkPointAction = () => { _monitor.startTryCheckPoint(storeCopyIdentifier); _checkPointer.tryCheckPoint(new SimpleTriggerInfo(triggerName)); _monitor.finishTryCheckPoint(storeCopyIdentifier); }; // Copy the store files long lastAppliedTransaction; StoreCopyCheckPointMutex mutex = _dataSource.StoreCopyCheckPointMutex; try { using (Resource @lock = mutex.StoreCopy(checkPointAction), ResourceIterator <StoreFileMetadata> files = _dataSource.listStoreFiles(includeLogs)) { lastAppliedTransaction = _checkPointer.lastCheckPointedTransactionId(); _monitor.startStreamingStoreFiles(storeCopyIdentifier); ByteBuffer temporaryBuffer = ByteBuffer.allocateDirect(( int )ByteUnit.mebiBytes(1)); while (Files.MoveNext()) { StoreFileMetadata meta = Files.Current; File file = meta.File(); bool isLogFile = meta.LogFile; int recordSize = meta.RecordSize(); using (ReadableByteChannel fileChannel = _fileSystem.open(file, OpenMode.READ)) { long fileSize = _fileSystem.getFileSize(file); DoWrite(writer, temporaryBuffer, file, recordSize, fileChannel, fileSize, storeCopyIdentifier, isLogFile); } } } } finally { _monitor.finishStreamingStoreFiles(storeCopyIdentifier); } return(anonymous(lastAppliedTransaction)); } catch (IOException e) { throw new ServerFailureException(e); } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public Void handle(org.neo4j.causalclustering.core.consensus.RaftMessages_NewEntry_Request newEntryRequest) throws Exception public override Void Handle(Org.Neo4j.causalclustering.core.consensus.RaftMessages_NewEntry_Request newEntryRequest) { BoundedNetworkWritableChannel sizeBoundChannel = new BoundedNetworkWritableChannel(Channel.byteBuf(), ByteUnit.gibiBytes(1)); Marshal.marshal(newEntryRequest.Content(), sizeBoundChannel); return(null); }