public void Repurpose(TopicPartition tp, int correlationId) { this.topicPartition = tp; this.bufferMessageCount = 0; this.bufferCurrentPos = 0; this.correlationId = correlationId; }
public Buffer GetTopicPartitionBuffer(TopicPartition tp, object freeForUseLockObj) { if (fillingUp.TryGetValue(tp, out Buffer buf)) { return(buf); } lock (freeForUseLockObj) { if (freeForUse.Count == 0) { if (this.bufferCount > 50) { return(null); } this.bufferCount += 1; this.freeForUse.Add(new Buffer(this.bufferSize)); } Buffer forUse = freeForUse[freeForUse.Count - 1]; freeForUse.RemoveAt(freeForUse.Count - 1); forUse.Repurpose(tp, ++this.lastCorrelationId); fillingUp.Add(tp, forUse); return(forUse); } }
public void Move_FillingUp_To_ForFinalize(TopicPartition tp) { var b = fillingUp[tp]; fillingUp.Remove(tp); forFinalize.Add(b); }
public unsafe void Produce(string topic, byte[] key, byte[] value) { var tp = new TopicPartition(topic, 0); var buffer = bufferPool.GetTopicPartitionBuffer_Blocking(tp, freeForUseLock); if (buffer.bufferCurrentPos + Buffer.MessageFixedOverhead + (key == null ? 0 : key.Length) + (value == null ? 0 : value.Length) > buffer.buffer.Length) { lock (producerLock) { this.bufferPool.Move_FillingUp_To_ForFinalize(tp); } buffer = bufferPool.GetTopicPartitionBuffer_Blocking(tp, freeForUseLock); } lock (producerLock) { fixed(byte *b = buffer.buffer) { if (buffer.bufferCurrentPos == 0) { byte *requestSizePtr; byte *messageSetSizePtr; byte *currentPosPtr = this.WriteProduceRequestHeader(b, topic, buffer.correlationId, out requestSizePtr, out messageSetSizePtr); buffer.requestSizeOffset = (int)(requestSizePtr - b); buffer.messageSetSizeOffset = (int)(messageSetSizePtr - b); byte *lengthPtr; byte *crcPtr; byte *recordCountPtr; byte *attributesPtr; currentPosPtr = this.WriteRecordBatchHeader(currentPosPtr, out lengthPtr, out crcPtr, out attributesPtr, out recordCountPtr); buffer.lengthOffset = (int)(lengthPtr - b); buffer.crcOffset = (int)(crcPtr - b); buffer.attributesOffset = (int)(attributesPtr - b); buffer.recordCountOffset = (int)(recordCountPtr - b); buffer.bufferCurrentPos = (int)(currentPosPtr - b); } buffer.bufferMessageCount += 1; byte *end = WriteRecord(b + buffer.bufferCurrentPos, key, value); buffer.bufferCurrentPos = (int)(end - b); } } }
public Buffer GetTopicPartitionBuffer_Blocking(TopicPartition tp, object freeForUseLockObj) { bool waited = false; while (true) { var result = GetTopicPartitionBuffer(tp, freeForUseLockObj); if (result != null) { if (waited) { // Console.WriteLine("blocked on buffer get"); } return(result); } waited = true; Task.Delay(TimeSpan.FromMilliseconds(50)).Wait(); } }