public void ShouldClaimRegionForZeroCopyEncoding() { int headerLength = _defaultHeader.Capacity; const int msgLength = 20; int frameLength = msgLength + headerLength; int alignedFrameLength = BitUtil.Align(frameLength, FrameDescriptor.FRAME_ALIGNMENT); const int tail = 0; BufferClaim bufferClaim = new BufferClaim(); A.CallTo(() => _termBuffer.PutIntOrdered(A <int> ._, A <int> ._)); _logMetaDataBuffer.PutLong(TermTailCounterOffset, TermAppender.Pack(TermID, tail)); Assert.That(_termAppender.Claim(_headerWriter, msgLength, bufferClaim), Is.EqualTo((long)alignedFrameLength)); Assert.That(bufferClaim.Offset, Is.EqualTo(tail + headerLength)); Assert.That(bufferClaim.Length, Is.EqualTo(msgLength)); // Map flyweight or encode to buffer directly then call commit() when done bufferClaim.Commit(); Assert.AreEqual(LogBufferDescriptor.RawTailVolatile(_logMetaDataBuffer, PartionIndex), TermAppender.Pack(TermID, tail + alignedFrameLength)); A.CallTo(() => _headerWriter.Write(_termBuffer, tail, frameLength, TermID)).MustHaveHappened(); }
public unsafe void CouldCreateLogBuffers() { var sw = new Stopwatch(); var l1 = new LogBuffers("../AeronLogBufferTests"); Assert.IsTrue(l1.TermLength >= LogBufferDescriptor.TERM_MIN_LENGTH); Console.WriteLine($"TermLength: {l1.TermLength}"); var activePartitionIndex = LogBufferDescriptor.ActivePartitionIndex(l1.LogMetaData); Console.WriteLine($"Active partition: {activePartitionIndex}"); var activePartition = l1.Partitions[activePartitionIndex]; var rawTail = activePartition.RawTailVolatile; Console.WriteLine($"Raw tail: {rawTail}"); var ta = new TermAppender(activePartition); var defaultHeader = DataHeaderFlyweight.CreateDefaultHeader(0, 0, activePartition.TermId); var headerWriter = new HeaderWriter(defaultHeader); BufferClaim claim; ta.Claim(headerWriter, 100, out claim); claim.Commit(); }
/// <inheritdoc /> public override long TryClaim(int length, BufferClaim bufferClaim) { CheckPayloadLength(length); long newPosition = CLOSED; if (!_isClosed) { long limit = _positionLimit.GetVolatile(); int termCount = LogBufferDescriptor.ActiveTermCount(_logMetaDataBuffer); TermAppender termAppender = _termAppenders[LogBufferDescriptor.IndexByTermCount(termCount)]; long rawTail = termAppender.RawTailVolatile(); long termOffset = rawTail & 0xFFFF_FFFFL; int termId = LogBufferDescriptor.TermId(rawTail); long position = LogBufferDescriptor.ComputeTermBeginPosition(termId, PositionBitsToShift, InitialTermId) + termOffset; if (termCount != (termId - InitialTermId)) { return(ADMIN_ACTION); } if (position < limit) { int resultingOffset = termAppender.Claim(_headerWriter, length, bufferClaim, termId); newPosition = NewPosition(termCount, (int)termOffset, termId, position, resultingOffset); } else { newPosition = BackPressureStatus(position, length); } } return(newPosition); }
internal ConcurrentPublication( ClientConductor clientConductor, string channel, int streamId, int sessionId, IReadablePosition positionLimit, int channelStatusId, LogBuffers logBuffers, long originalRegistrationId, long registrationId) : base( clientConductor, channel, streamId, sessionId, positionLimit, channelStatusId, logBuffers, originalRegistrationId, registrationId ) { var buffers = logBuffers.DuplicateTermBuffers(); for (var i = 0; i < LogBufferDescriptor.PARTITION_COUNT; i++) { _termAppenders[i] = new TermAppender(buffers[i], _logMetaDataBuffer, i); } }
internal Publication(ClientConductor clientConductor, string channel, int streamId, int sessionId, IReadablePosition positionLimit, LogBuffers logBuffers, long registrationId) { var buffers = logBuffers.AtomicBuffers(); var logMetaDataBuffer = buffers[LogBufferDescriptor.LOG_META_DATA_SECTION_INDEX]; for (var i = 0; i < LogBufferDescriptor.PARTITION_COUNT; i++) { _termAppenders[i] = new TermAppender(buffers[i], buffers[i + LogBufferDescriptor.PARTITION_COUNT]); } var termLength = logBuffers.TermLength(); _maxPayloadLength = LogBufferDescriptor.MtuLength(logMetaDataBuffer) - DataHeaderFlyweight.HEADER_LENGTH; MaxMessageLength = FrameDescriptor.ComputeMaxMessageLength(termLength); _clientConductor = clientConductor; Channel = channel; StreamId = streamId; SessionId = sessionId; InitialTermId = LogBufferDescriptor.InitialTermId(logMetaDataBuffer); _logMetaDataBuffer = logMetaDataBuffer; RegistrationId = registrationId; _positionLimit = positionLimit; _logBuffers = logBuffers; _positionBitsToShift = IntUtil.NumberOfTrailingZeros(termLength); _headerWriter = new HeaderWriter(LogBufferDescriptor.DefaultFrameHeader(logMetaDataBuffer)); }
public void ShouldAppendFrameTwiceToLog() { int headerLength = _defaultHeader.Capacity; UnsafeBuffer buffer = new UnsafeBuffer(new byte[128]); const int msgLength = 20; int frameLength = msgLength + headerLength; int alignedFrameLength = BitUtil.Align(frameLength, FrameDescriptor.FRAME_ALIGNMENT); int tail = 0; _logMetaDataBuffer.PutLong(TermTailCounterOffset, TermAppender.Pack(TermID, tail)); Assert.That(_termAppender.AppendUnfragmentedMessage(_headerWriter, buffer, 0, msgLength, RVS), Is.EqualTo((long)alignedFrameLength)); Assert.That(_termAppender.AppendUnfragmentedMessage(_headerWriter, buffer, 0, msgLength, RVS), Is.EqualTo((long)alignedFrameLength * 2)); Assert.AreEqual(LogBufferDescriptor.RawTailVolatile(_logMetaDataBuffer, PartionIndex), TermAppender.Pack(TermID, tail + alignedFrameLength * 2)); A.CallTo(() => _headerWriter.Write(_termBuffer, tail, frameLength, TermID)).MustHaveHappened() .Then(A.CallTo(() => _termBuffer.PutBytes(headerLength, buffer, 0, msgLength)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutLong(tail + DataHeaderFlyweight.RESERVED_VALUE_OFFSET, RV)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(tail, frameLength)).MustHaveHappened()) .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, alignedFrameLength, frameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutBytes(alignedFrameLength + headerLength, buffer, 0, msgLength)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutLong(alignedFrameLength + DataHeaderFlyweight.RESERVED_VALUE_OFFSET, RV)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(alignedFrameLength, frameLength)).MustHaveHappened()); }
public void ShouldAppendFrameTwiceToLog() { int headerLength = _defaultHeader.Capacity; UnsafeBuffer buffer = new UnsafeBuffer(new byte[128]); const int msgLength = 20; int frameLength = msgLength + headerLength; int alignedFrameLength = BitUtil.Align(frameLength, FrameDescriptor.FRAME_ALIGNMENT); int tail = 0; A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, alignedFrameLength)) .ReturnsNextFromSequence(TermAppender.Pack(TermID, tail), TermAppender.Pack(TermID, alignedFrameLength)); Assert.That(_termAppender.AppendUnfragmentedMessage(_headerWriter, buffer, 0, msgLength), Is.EqualTo((long)alignedFrameLength)); Assert.That(_termAppender.AppendUnfragmentedMessage(_headerWriter, buffer, 0, msgLength), Is.EqualTo((long)alignedFrameLength * 2)); A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, alignedFrameLength)).MustHaveHappened() .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, tail, frameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutBytes(headerLength, buffer, 0, msgLength)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(tail, frameLength)).MustHaveHappened()) .Then(A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, alignedFrameLength)).MustHaveHappened()) .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, alignedFrameLength, frameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutBytes(alignedFrameLength + headerLength, buffer, 0, msgLength)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(alignedFrameLength, frameLength)).MustHaveHappened()); }
public void ShouldFragmentMessageOverTwoFrames() { int msgLength = MaxPayloadLength + 1; int headerLength = _defaultHeader.Capacity; int frameLength = headerLength + 1; int requiredCapacity = BitUtil.Align(headerLength + 1, FrameDescriptor.FRAME_ALIGNMENT) + MaxFrameLength; UnsafeBuffer buffer = new UnsafeBuffer(new byte[msgLength]); int tail = 0; _logMetaDataBuffer.PutLong(TermTailCounterOffset, TermAppender.Pack(TermID, tail)); Assert.That(_termAppender.AppendFragmentedMessage(_headerWriter, buffer, 0, msgLength, MaxPayloadLength, RVS), Is.EqualTo((long)requiredCapacity)); Assert.AreEqual(LogBufferDescriptor.RawTailVolatile(_logMetaDataBuffer, PartionIndex), TermAppender.Pack(TermID, tail + requiredCapacity)); A.CallTo(() => _headerWriter.Write(_termBuffer, tail, MaxFrameLength, TermID)).MustHaveHappened() .Then(A.CallTo(() => _termBuffer.PutBytes(tail + headerLength, buffer, 0, MaxPayloadLength)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutByte(FrameDescriptor.FlagsOffset(tail), FrameDescriptor.BEGIN_FRAG_FLAG)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutLong(tail + DataHeaderFlyweight.RESERVED_VALUE_OFFSET, RV)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(tail, MaxFrameLength)).MustHaveHappened()) .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, MaxFrameLength, frameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutBytes(MaxFrameLength + headerLength, buffer, MaxPayloadLength, 1)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutByte(FrameDescriptor.FlagsOffset(MaxFrameLength), FrameDescriptor.END_FRAG_FLAG)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutLong(MaxFrameLength + DataHeaderFlyweight.RESERVED_VALUE_OFFSET, RV)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(MaxFrameLength, frameLength)).MustHaveHappened()); }
/// <inheritdoc /> public override long Offer(IDirectBuffer buffer, int offset, int length, ReservedValueSupplier reservedValueSupplier = null) { long newPosition = CLOSED; if (!_isClosed) { long limit = _positionLimit.GetVolatile(); int termCount = LogBufferDescriptor.ActiveTermCount(_logMetaDataBuffer); TermAppender termAppender = _termAppenders[LogBufferDescriptor.IndexByTermCount(termCount)]; long rawTail = termAppender.RawTailVolatile(); long termOffset = rawTail & 0xFFFF_FFFFL; int termId = LogBufferDescriptor.TermId(rawTail); long position = LogBufferDescriptor.ComputeTermBeginPosition(termId, PositionBitsToShift, InitialTermId) + termOffset; if (termCount != (termId - InitialTermId)) { return(ADMIN_ACTION); } if (position < limit) { int resultingOffset; if (length <= MaxPayloadLength) { CheckPositiveLength(length); resultingOffset = termAppender.AppendUnfragmentedMessage( _headerWriter, buffer, offset, length, reservedValueSupplier, termId); } else { CheckMaxMessageLength(length); resultingOffset = termAppender.AppendFragmentedMessage( _headerWriter, buffer, offset, length, MaxPayloadLength, reservedValueSupplier, termId); } newPosition = NewPosition(termCount, (int)termOffset, termId, position, resultingOffset); } else { newPosition = BackPressureStatus(position, length); } } return(newPosition); }
public void ShouldPackResult() { const int termId = 7; const int termOffset = -1; long result = TermAppender.Pack(termId, termOffset); Assert.That(TermAppender.TermId(result), Is.EqualTo(termId)); Assert.That(TermAppender.TermOffset(result), Is.EqualTo(termOffset)); }
public void SetUp() { _defaultHeader = new UnsafeBuffer(new byte[DataHeaderFlyweight.HEADER_LENGTH]); _termBuffer = A.Fake <IAtomicBuffer>(x => x.Wrapping(new UnsafeBuffer(new byte[TermBufferLength]))); _metaDataBuffer = A.Fake <IAtomicBuffer>(); _headerWriter = A.Fake <HeaderWriter>(x => x.Wrapping(new HeaderWriter(DataHeaderFlyweight.CreateDefaultHeader(0, 0, TermID)))); A.CallTo(() => _termBuffer.Capacity).Returns(TermBufferLength); A.CallTo(() => _metaDataBuffer.Capacity).Returns(MetaDataBufferLength); _termAppender = new TermAppender(_termBuffer, _metaDataBuffer); }
public void SetUp() { _defaultHeader = new UnsafeBuffer(new byte[DataHeaderFlyweight.HEADER_LENGTH]); var buffer = new UnsafeBuffer(new byte[TermBufferLength]); _termBuffer = A.Fake <UnsafeBuffer>(x => x.Wrapping(buffer)); _logMetaDataBuffer = A.Fake <UnsafeBuffer>(x => x.Wrapping(new UnsafeBuffer(new byte[MetaDataBufferLength]))); _headerWriter = A.Fake <HeaderWriter>(x => x.Wrapping(new HeaderWriter(DataHeaderFlyweight.CreateDefaultHeader(0, 0, TermID)))); A.CallTo(() => _termBuffer.Capacity).Returns(TermBufferLength); A.CallTo(() => _termBuffer.BufferPointer).Returns(buffer.BufferPointer); A.CallTo(() => _logMetaDataBuffer.Capacity).Returns(MetaDataBufferLength); _termAppender = new TermAppender(_termBuffer, _logMetaDataBuffer, PartionIndex); }
private long NewPosition(int index, int currentTail, long position, long result) { var newPosition = ADMIN_ACTION; var termOffset = TermAppender.TermOffset(result); if (termOffset > 0) { newPosition = (position - currentTail) + termOffset; } else if (termOffset == TermAppender.TRIPPED) { var nextIndex = LogBufferDescriptor.NextPartitionIndex(index); _termAppenders[nextIndex].TailTermId(TermAppender.TermId(result) + 1); LogBufferDescriptor.ActivePartitionIndexOrdered(_logMetaDataBuffer, nextIndex); } return(newPosition); }
public void ShouldPadLogAndTripWhenAppendingWithInsufficientRemainingCapacity() { const int msgLength = 120; int headerLength = _defaultHeader.Capacity; int requiredFrameSize = BitUtil.Align(headerLength + msgLength, FrameDescriptor.FRAME_ALIGNMENT); int tailValue = TermBufferLength - BitUtil.Align(msgLength, FrameDescriptor.FRAME_ALIGNMENT); UnsafeBuffer buffer = new UnsafeBuffer(new byte[128]); int frameLength = TermBufferLength - tailValue; A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, requiredFrameSize)).Returns(TermAppender.Pack(TermID, tailValue)); long expectResult = TermAppender.Pack(TermID, TermAppender.TRIPPED); Assert.That(_termAppender.AppendUnfragmentedMessage(_headerWriter, buffer, 0, msgLength), Is.EqualTo(expectResult)); A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, requiredFrameSize)).MustHaveHappened() .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, tailValue, frameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutShort(FrameDescriptor.TypeOffset(tailValue), (short)FrameDescriptor.PADDING_FRAME_TYPE)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(tailValue, frameLength)).MustHaveHappened()); }
private long NewPosition(int index, int currentTail, long position, long result) { long newPosition = TermAppender.TRIPPED; int termOffset = TermAppender.TermOffset(result); if (termOffset > 0) { newPosition = (position - currentTail) + termOffset; } else if (termOffset == TermAppender.TRIPPED) { //int nextIndex = NextPartitionIndex(index); //int nextNextIndex = NextPartitionIndex(nextIndex); //_termAppenders[nextIndex].TailTermId(TermAppender.TermId(result) + 1); //_termAppenders[nextNextIndex].StatusOrdered(LogBufferDescriptor.NEEDS_CLEANING); //ActivePartitionIndex(_logBuffers.LogMetaData, nextIndex); RotateLog(_logBuffers.Partitions, _logBuffers.LogMetaData, index, TermAppender.TermId(result) + 1); _cleanEvent.Set(); } return(newPosition); }
public void ShouldClaimRegionForZeroCopyEncoding() { int headerLength = _defaultHeader.Capacity; const int msgLength = 20; int frameLength = msgLength + headerLength; int alignedFrameLength = BitUtil.Align(frameLength, FrameDescriptor.FRAME_ALIGNMENT); const int tail = 0; BufferClaim bufferClaim = new BufferClaim(); A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, alignedFrameLength)) .Returns(TermAppender.Pack(TermID, tail)); Assert.That(_termAppender.Claim(_headerWriter, msgLength, bufferClaim), Is.EqualTo((long)alignedFrameLength)); Assert.That(bufferClaim.Offset, Is.EqualTo(tail + headerLength)); Assert.That(bufferClaim.Length, Is.EqualTo(msgLength)); // Map flyweight or encode to buffer directly then call commit() when done bufferClaim.Commit(); A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, alignedFrameLength)).MustHaveHappened() .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, tail, frameLength, TermID)).MustHaveHappened()); }
public void ShouldFragmentMessageOverTwoFrames() { int msgLength = MaxPayloadLength + 1; int headerLength = _defaultHeader.Capacity; int frameLength = headerLength + 1; int requiredCapacity = BitUtil.Align(headerLength + 1, FrameDescriptor.FRAME_ALIGNMENT) + MaxFrameLength; UnsafeBuffer buffer = new UnsafeBuffer(new byte[msgLength]); int tail = 0; A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, requiredCapacity)).Returns(TermAppender.Pack(TermID, tail)); Assert.That(_termAppender.AppendFragmentedMessage(_headerWriter, buffer, 0, msgLength, MaxPayloadLength), Is.EqualTo((long)requiredCapacity)); A.CallTo(() => _metaDataBuffer.GetAndAddLong(LogBufferDescriptor.TERM_TAIL_COUNTER_OFFSET, requiredCapacity)).MustHaveHappened() .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, tail, MaxFrameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutBytes(tail + headerLength, buffer, 0, MaxPayloadLength)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutByte(FrameDescriptor.FlagsOffset(tail), FrameDescriptor.BEGIN_FRAG_FLAG)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(tail, MaxFrameLength)).MustHaveHappened()) .Then(A.CallTo(() => _headerWriter.Write(_termBuffer, MaxFrameLength, frameLength, TermID)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutBytes(MaxFrameLength + headerLength, buffer, MaxPayloadLength, 1)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutByte(FrameDescriptor.FlagsOffset(MaxFrameLength), FrameDescriptor.END_FRAG_FLAG)).MustHaveHappened()) .Then(A.CallTo(() => _termBuffer.PutIntOrdered(MaxFrameLength, frameLength)).MustHaveHappened()); }
public AppendLog(string filepath, int bufferSizeMb = 100) { var bufferSizeInBytes = BitUtil.FindNextPositivePowerOfTwo(bufferSizeMb * 1024 * 1024); _logBuffers = new LogBuffers(filepath, bufferSizeInBytes); for (int i = 0; i < PARTITION_COUNT; i++) { _termAppenders[i] = new TermAppender(_logBuffers.Buffers[i], _logBuffers.Buffers[i + PARTITION_COUNT]); } _termLengthMask = _logBuffers.TermLength - 1; _positionBitsToShift = BitUtil.NumberOfTrailingZeros(_logBuffers.TermLength); _initialTermId = InitialTermId(_logBuffers.LogMetaData); //var defaultHeader = DefaultFrameHeader(_logBuffers.LogMetaData); var defaultHeader = DataHeaderFlyweight.CreateDefaultHeader(0, 0, _initialTermId); _headerWriter = new HeaderWriter(defaultHeader); _subscriberPosition = Position; Trace.Assert(_subscriberPosition == Position); _cleaner = Task.Factory.StartNew(async() => { try { } finally { while (!_cts.IsCancellationRequested) { // try to clean every second await _cleanEvent.WaitAsync(1000); CleanLogBuffer(); } } }, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default) .ContinueWith(task => { Console.WriteLine("AppendLog CleanLogBuffer should never throw exceptions" + Environment.NewLine + task.Exception); Environment.FailFast("AppendLog CleanLogBuffer should never throw exceptions", task.Exception); }, TaskContinuationOptions.OnlyOnFaulted); _poller = Task.Factory.StartNew(() => { try { } finally { var sw = new SpinWait(); while (!_cts.IsCancellationRequested) { var fragments = Poll(); if (fragments > 0) { sw.Reset(); } else { sw.SpinOnce(); } // TODO try waithandle as in IpcLongIncrementListener //Thread.SpinWait(0); } } }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default) .ContinueWith(task => { Console.WriteLine("AppendLog Poll should never throw exceptions" + Environment.NewLine + task.Exception); Environment.FailFast("AppendLog Poll should never throw exceptions", task.Exception); }, TaskContinuationOptions.OnlyOnFaulted); }