public void OpenCloseDirectFiles() { var path = TestUtils.GetPath("UtilsTests"); var count = 10_000; using (Benchmark.Run("Open/Close view", count * 1_000)) { using (var directFile = new DirectFile(Path.Combine(path, "test_file.bin"), 4096, true)) { for (int i = 1; i <= count; i++) { directFile.Grow(4096 + 4096 * i); directFile.Buffer.Write <long>(4096 * i - 8, i); } } } Benchmark.Dump(); using (var directFile = new DirectFile(Path.Combine(path, "test_file.bin"), 4096, true)) { for (int i = 1; i <= count; i++) { directFile.Grow(4096 + 4096 * i); var val = directFile.Buffer.Read <long>(4096 * i - 8); Assert.AreEqual(i, val); } } }
public unsafe LogBuffer(string filename, int termSizeMb = 5) { Filename = filename; TermSize = termSizeMb * 1024 * 1024; _df = new DirectFile(filename, LogSize); _writerTerm = ActiveTermId; _writerTailPtr = Tail(_writerTerm); _writerTermPtr = _df._buffer._data + HeaderSize + TermSize * _writerTerm; // init at the last known place _readerTerm = _writerTerm; _readerTail = *(int *)_writerTailPtr; _readerTermPtr = _writerTermPtr; _cts = new CancellationTokenSource(); _reader = Task.Factory.StartNew(() => { var sw = new SpinWait(); while (!_cts.Token.IsCancellationRequested) { try { } finally { var tail = _readerTail; // Volatile.Read(ref *(int*)_readerTailPtr); if (tail >= TermSize || *(int *)(_readerTermPtr + tail) == -1) { while (ActiveTermId == _readerTerm) { sw.SpinOnce(); } // switch term _readerTerm = (_readerTerm + 1) % NumberOfTerms; _readerTail = 0; _readerTermPtr = _df._buffer._data + HeaderSize + TermSize * _readerTerm; } else { var len = *(int *)(_readerTermPtr + tail); if (len > 0) { // could read value byte[] bytes = new byte[len]; Marshal.Copy((_readerTermPtr + tail + 4), bytes, 0, len); OnAppend?.Invoke((_readerTermPtr + tail)); _readerTail = _readerTail + len + 4; } } Thread.SpinWait(1); // TODO? implement signaling via WaitHandle? //if (sw.NextSpinWillYield) sw.Reset(); //sw.SpinOnce(); } } OptimizationSettings.TraceVerbose("LogBuffer invoke loop exited"); }, _cts.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default) .ContinueWith(task => { Console.WriteLine("LogBuffer OnAppendHandlerOld Invoke should never throw exceptions" + Environment.NewLine + task.Exception); Environment.FailFast("LogBuffer OnAppendHandlerOld Invoke should never throw exceptions", task.Exception); }, TaskContinuationOptions.OnlyOnFaulted); }
public void CouldOpenSparse() { var path = TestUtils.GetPath(); var df = new DirectFile(Path.Combine(path, "test.file"), 64L * 1024 * 1024 * 1024, true, FileOptions.WriteThrough, sparse: true); Unsafe.InitBlockUnaligned(df.DirectBuffer.Data, 0, 1024); df.Dispose(); }
public void OpenHugeDirectFile() { var path = TestUtils.GetPath(); var filePath = Path.Combine(path, "test.tmp"); var df = new DirectFile(filePath, 50 * 1024 * 1024 * 1024L, true); var span = df.Buffer.Span; span[1] = 1; df.Dispose(); }
public void CouldWriteToWriteThroughFile() { var path = TestUtils.GetPath(); var df = new DirectFile(Path.Combine(path, "test.file"), 1024 * 1024, true, FileOptions.WriteThrough); df.DirectBuffer.Write(0, 123L); var bytes = new byte[] { 123 }; df._va.WriteArray(0, bytes, 0, 1); var ums = df._mmf.CreateViewStream(0, 1024 * 1024); ums.Write(bytes, 0, 1); ums.Dispose(); df.Flush(true); df.Dispose(); }
public void CouldReadUnallocatedSparse() { var path = TestUtils.GetPath(); var df = new DirectFile(Path.Combine(path, "test.file"), 1L * 1024 * 1024, true, FileOptions.WriteThrough, sparse: true); var sum = 0; for (long i = 0; i < df.Length; i++) { sum += unchecked (df.Buffer[i]); // Console.WriteLine(df.Buffer[i]); } Assert.AreEqual(0, sum); df.Dispose(); }
public IpcLongIncrementListener(string filename, Action <long, long> action, long init = -1L) { _action = action; _df = new DirectFile(filename + ".ipclistener", 8); var handleName = Path.GetFileName(filename) + ".ipclistener"; bool created; _eh = new EventWaitHandle(false, EventResetMode.ManualReset, handleName, out created); if (created && init != -1L) { _df.Buffer.VolatileWriteInt64(0, init); _lastSeenValue = init; } else { _lastSeenValue = _df.Buffer.VolatileReadInt64(0); } }
public LogBuffers(string logFileName, int termLength = LogBufferDescriptor.TERM_MIN_LENGTH) { try { long logLength = LogBufferDescriptor.PARTITION_COUNT * (LogBufferDescriptor.TERM_META_DATA_LENGTH + termLength) + LogBufferDescriptor.LOG_META_DATA_LENGTH; termLength = LogBufferDescriptor.ComputeTermLength(logLength); LogBufferDescriptor.CheckTermLength(termLength); _df = new DirectFile(logFileName, logLength); _termLength = termLength; // if log length exceeds MAX_INT we need multiple mapped buffers, (see FileChannel.map doc). if (logLength < int.MaxValue) { int metaDataSectionOffset = termLength * LogBufferDescriptor.PARTITION_COUNT; for (int i = 0; i < LogBufferDescriptor.PARTITION_COUNT; i++) { int metaDataOffset = metaDataSectionOffset + (i * LogBufferDescriptor.TERM_META_DATA_LENGTH); _buffers[i] = new DirectBuffer(termLength, _df.Buffer.Data + i * termLength); _buffers[i + LogBufferDescriptor.PARTITION_COUNT] = new DirectBuffer(LogBufferDescriptor.TERM_META_DATA_LENGTH, _df.Buffer.Data + metaDataOffset); _partitions[i] = new LogBufferPartition(_buffers[i], _buffers[i + LogBufferDescriptor.PARTITION_COUNT]); } _buffers[_buffers.Length - 1] = new DirectBuffer(LogBufferDescriptor.LOG_META_DATA_LENGTH, _df.Buffer.Data + (int)(logLength - LogBufferDescriptor.LOG_META_DATA_LENGTH)); } else { throw new NotImplementedException("TODO Check .NET mapping limit"); } } catch (IOException ex) { throw new AggregateException(ex); } foreach (var buffer in _buffers) { buffer.VerifyAlignment(8); } }
public unsafe void CouldCatchErrorWhileWritingPastBoundary() { // This doesn;t throw unless size is 4096 vs 12 // Could not make any assumptions that it is safe to write past boundary // with try..catch. Even byte[] allows it and we probably corrupt // .NETs memory next to it. var bytes = new byte[12]; var fb = new FixedBuffer(bytes); fixed(byte *ptr = &bytes[10]) { *(long *)(ptr) = long.MaxValue; } var df = new DirectFile("../CouldCatchErrorWhileWritingPastBoundary", 12); *(long *)(df.Buffer.Data + 10) = long.MaxValue; df.Dispose(); var df2 = new DirectFile("../CouldCatchErrorWhileWritingPastBoundary", 12); Assert.AreEqual(long.MaxValue, *(long *)(df2.Buffer.Data + 10)); }
private PersistentArray(string filename, long minCapacity, T fill) { _df = new DirectFile(filename, DataOffset + minCapacity * ItemSize); }
public void FileConcurrentWriteRead() { var pagesPerWrite = new[] { 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024 }; var pageSize = 512; var count = TestUtils.GetBenchCount(5_000L, 50); var path = TestUtils.GetPath(clear: true); var file = Path.Combine(path, "test.file"); using (var fs = new FileStream(file, FileMode.CreateNew)) { fs.SetLength(count * pageSize); } foreach (var mult in pagesPerWrite) { var sem = new SemaphoreSlim(0, (int)count); var page = new byte[pageSize * mult]; var rt = Task.Run(() => { using (var df = new DirectFile(file, 0L, false, FileOptions.SequentialScan)) //using (var fs2 = new FileStream(file, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, // pageSize * mult, // FileOptions.None)) //using (var memoryMappedFile = // MemoryMappedFile.CreateFromFile(fs2, Guid.NewGuid().ToString(), count * pageSize, // MemoryMappedFileAccess.ReadWrite, // HandleInheritability.None, false)) //using (var memoryMappedViewAccessor = memoryMappedFile.CreateViewAccessor()) { using (Benchmark.Run("Read " + mult, count * pageSize)) { var i = 0; var page2 = new byte[pageSize * mult]; while (i < count / mult) { sem.Wait(); var sp = df.DirectBuffer.Span.Slice(pageSize * i * mult, pageSize * mult); //memoryMappedViewAccessor.ReadArray(pageSize * i * (long)mult, page2, 0, // pageSize * mult); if (!((ReadOnlySpan <byte>)sp).SequenceEqual(page)) { Assert.Fail("Pages are not equal"); } i++; } } } }); using (Benchmark.Run("Write " + mult, count * pageSize)) { using (var fs = new FileStream(file, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, pageSize * mult, FileOptions.WriteThrough)) //| WAL.WALWriter.FILE_FLAG_NO_BUFFERING)) // { fs.SetLength(count * pageSize); new Random(123).NextBytes(page); for (int i = 0; i < count / mult; i++) { fs.Position = pageSize * i * (long)mult; fs.Write(page, 0, page.Length); sem.Release(); } } } rt.Wait(); } Benchmark.Dump(); }