public BrokerSetting(string chunkFileStoreRootPath = @"c:\equeue-store", int messageChunkDataSize = 256 * 1024 * 1024, int chunkFlushInterval = 100, int chunkCacheMaxPercent = 75, int chunkCacheMinPercent = 40, int maxLogRecordSize = 5 * 1024 * 1024, int chunkWriteBuffer = 128 * 1024, int chunkReadBuffer = 128 * 1024, bool syncFlush = false, bool enableCache = true) { ProducerAddress = new IPEndPoint(SocketUtils.GetLocalIPV4(), 5000); ConsumerAddress = new IPEndPoint(SocketUtils.GetLocalIPV4(), 5001); AdminAddress = new IPEndPoint(SocketUtils.GetLocalIPV4(), 5002); NotifyWhenMessageArrived = true; DeleteMessagesInterval = 1000 * 10; DeleteQueueMessagesInterval = 1000 * 10; PersistConsumeOffsetInterval = 1000 * 1; CheckBlockingPullRequestMilliseconds = 1000 * 1; ProducerExpiredTimeout = 1000 * 30; ConsumerExpiredTimeout = 1000 * 30; RemoveConsumerWhenDisconnect = false; AutoCreateTopic = true; TopicDefaultQueueCount = 4; TopicMaxQueueCount = 64; MessageMaxSize = 1024 * 1024 * 4; FileStoreRootPath = chunkFileStoreRootPath; MessageChunkConfig = new ChunkManagerConfig( Path.Combine(chunkFileStoreRootPath, @"message-chunks"), new DefaultFileNamingStrategy("message-chunk-"), messageChunkDataSize, 0, 0, chunkFlushInterval, enableCache, syncFlush, Environment.ProcessorCount * 2, maxLogRecordSize, chunkWriteBuffer, chunkReadBuffer, chunkCacheMaxPercent, chunkCacheMinPercent, 1, 5, 300000, true, true); QueueChunkConfig = new ChunkManagerConfig( Path.Combine(chunkFileStoreRootPath, @"queue-chunks"), new DefaultFileNamingStrategy("queue-chunk-"), 0, 12, 1000000, chunkFlushInterval, enableCache, syncFlush, Environment.ProcessorCount * 2, 12, chunkWriteBuffer, chunkReadBuffer, chunkCacheMaxPercent, chunkCacheMinPercent, 1, 5, 1000000, false, false); }
private Chunk(string filename, ChunkManager chunkManager, ChunkManagerConfig chunkConfig, bool isMemoryChunk) { Ensure.NotNullOrEmpty(filename, "filename"); Ensure.NotNull(chunkManager, "chunkManager"); Ensure.NotNull(chunkConfig, "chunkConfig"); _filename = filename; _chunkManager = chunkManager; _chunkConfig = chunkConfig; _isMemoryChunk = isMemoryChunk; _lastActiveTime = DateTime.Now; }
public ChunkManager(string name, ChunkManagerConfig config, string relativePath = null) { Ensure.NotNull(name, "name"); Ensure.NotNull(config, "config"); Name = name; _config = config; if (string.IsNullOrEmpty(relativePath)) { _chunkPath = _config.BasePath; } else { _chunkPath = Path.Combine(_config.BasePath, relativePath); } _chunks = new ConcurrentDictionary <int, Chunk>(); _scheduleService = ObjectContainer.Resolve <IScheduleService>(); }
public static Chunk FromOngoingFile <T>(string filename, ChunkManager chunkManager, ChunkManagerConfig config, Func <byte[], T> readRecordFunc, bool isMemoryChunk) where T : ILogRecord { var chunk = new Chunk(filename, chunkManager, config, isMemoryChunk); try { chunk.InitOngoing(readRecordFunc); } catch (OutOfMemoryException) { chunk.Dispose(); throw; } catch (Exception ex) { _logger.Error(string.Format("Chunk {0} init from ongoing file failed.", chunk), ex); chunk.Dispose(); throw; } return(chunk); }
public static Chunk FromCompletedFile(string filename, ChunkManager chunkManager, ChunkManagerConfig config, bool isMemoryChunk) { var chunk = new Chunk(filename, chunkManager, config, isMemoryChunk); try { chunk.InitCompleted(); } catch (OutOfMemoryException) { chunk.Dispose(); throw; } catch (Exception ex) { _logger.Error(string.Format("Chunk {0} init from completed file failed.", chunk), ex); chunk.Dispose(); throw; } return(chunk); }
public static Chunk CreateNew(string filename, int chunkNumber, ChunkManager chunkManager, ChunkManagerConfig config, bool isMemoryChunk) { var chunk = new Chunk(filename, chunkManager, config, isMemoryChunk); try { chunk.InitNew(chunkNumber); } catch (OutOfMemoryException) { chunk.Dispose(); throw; } catch (Exception ex) { _logger.Error(string.Format("Chunk {0} create failed.", chunk), ex); chunk.Dispose(); throw; } return(chunk); }
public BrokerSetting(bool isMessageStoreMemoryMode = false, string chunkFileStoreRootPath = @"c:\equeue-store", int messageChunkDataSize = 1024 * 1024 * 1024, int chunkFlushInterval = 100, int chunkCacheMaxPercent = 75, int chunkCacheMinPercent = 40, int maxLogRecordSize = 5 * 1024 * 1024, int chunkWriteBuffer = 128 * 1024, int chunkReadBuffer = 128 * 1024, bool syncFlush = false, bool enableCache = true, int messageChunkLocalCacheSize = 300000, int queueChunkLocalCacheSize = 10000) { BrokerInfo = new BrokerInfo( "DefaultBroker", "DefaultGroup", "DefaultCluster", BrokerRole.Master, new IPEndPoint(SocketUtils.GetLocalIPV4(), 5000).ToAddress(), new IPEndPoint(SocketUtils.GetLocalIPV4(), 5001).ToAddress(), new IPEndPoint(SocketUtils.GetLocalIPV4(), 5002).ToAddress()); NameServerList = new List<IPEndPoint>() { new IPEndPoint(SocketUtils.GetLocalIPV4(), 9493) }; NotifyWhenMessageArrived = true; RegisterBrokerToNameServerInterval = 1000 * 5; DeleteMessagesInterval = 1000 * 10; DeleteQueueMessagesInterval = 1000 * 10; DeleteMessageIgnoreUnConsumed = true; PersistConsumeOffsetInterval = 1000 * 1; CheckBlockingPullRequestMilliseconds = 1000 * 1; ProducerExpiredTimeout = 1000 * 10; ConsumerExpiredTimeout = 1000 * 10; RemoveConsumerWhenDisconnect = true; AutoCreateTopic = true; TopicDefaultQueueCount = 4; TopicMaxQueueCount = 256; MessageMaxSize = 1024 * 1024 * 4; MessageWriteQueueThreshold = 2 * 10000; IsMessageStoreMemoryMode = isMessageStoreMemoryMode; FileStoreRootPath = chunkFileStoreRootPath; LatestMessageShowCount = 100; MessageChunkConfig = new ChunkManagerConfig( Path.Combine(chunkFileStoreRootPath, @"message-chunks"), new DefaultFileNamingStrategy("message-chunk-"), messageChunkDataSize, 0, 0, chunkFlushInterval, enableCache, syncFlush, Environment.ProcessorCount * 8, maxLogRecordSize, chunkWriteBuffer, chunkReadBuffer, chunkCacheMaxPercent, chunkCacheMinPercent, 1, 5, messageChunkLocalCacheSize, true, true); QueueChunkConfig = new ChunkManagerConfig( Path.Combine(chunkFileStoreRootPath, @"queue-chunks"), new DefaultFileNamingStrategy("queue-chunk-"), 0, 12, 1000000, chunkFlushInterval, enableCache, syncFlush, Environment.ProcessorCount * 2, 12, chunkWriteBuffer, chunkReadBuffer, chunkCacheMaxPercent, chunkCacheMinPercent, 1, 5, queueChunkLocalCacheSize, false, false); }