public BrokerSetting(string chunkFileStoreRootPath = @"c:\equeue-store", int messageChunkDataSize = 256 * 1024 * 1024, int chunkFlushInterval = 100, int chunkCacheMaxPercent = 75, int chunkCacheMinPercent = 40, int maxLogRecordSize = 5 * 1024 * 1024, int chunkWriteBuffer = 128 * 1024, int chunkReadBuffer = 128 * 1024, bool syncFlush = false, bool enableCache = true)
{
ProducerAddress = new IPEndPoint(SocketUtils.GetLocalIPV4(), 5000);
ConsumerAddress = new IPEndPoint(SocketUtils.GetLocalIPV4(), 5001);
AdminAddress = new IPEndPoint(SocketUtils.GetLocalIPV4(), 5002);
NotifyWhenMessageArrived = true;
DeleteMessagesInterval = 1000 * 10;
DeleteQueueMessagesInterval = 1000 * 10;
PersistConsumeOffsetInterval = 1000 * 1;
CheckBlockingPullRequestMilliseconds = 1000 * 1;
ProducerExpiredTimeout = 1000 * 30;
ConsumerExpiredTimeout = 1000 * 30;
RemoveConsumerWhenDisconnect = false;
AutoCreateTopic = true;
TopicDefaultQueueCount = 4;
TopicMaxQueueCount = 64;
MessageMaxSize = 1024 * 1024 * 4;
FileStoreRootPath = chunkFileStoreRootPath;
MessageChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, @"message-chunks"),
new DefaultFileNamingStrategy("message-chunk-"),
messageChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
Environment.ProcessorCount * 2,
maxLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxPercent,
chunkCacheMinPercent,
1,
5,
300000,
true,
true);
QueueChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, @"queue-chunks"),
new DefaultFileNamingStrategy("queue-chunk-"),
0,
12,
1000000,
chunkFlushInterval,
enableCache,
syncFlush,
Environment.ProcessorCount * 2,
12,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxPercent,
chunkCacheMinPercent,
1,
5,
1000000,
false,
false);
}
public void Load()
{
var path = @"c:\estore-files\event-chunks";
var chunkDataSize = 512 * 1024 * 1024;
var maxLogRecordSize = 4 * 1024 * 1024;
var config = new ChunkManagerConfig(
path,
new DefaultFileNamingStrategy("event-chunk-"),
chunkDataSize,
0,
0,
100,
false,
false,
Environment.ProcessorCount * 2,
maxLogRecordSize,
128 * 1024,
128 * 1024,
90,
45,
1,
5,
1000000,
false);
_chunkManager = new ChunkManager("EventChunk", config, false);
_chunkWriter = new ChunkWriter(_chunkManager);
}
static void Main(string[] args)
{
var configuration = Configuration
.Create()
.UseAutofac()
.RegisterCommonComponents()
.UseLog4Net()
.UseJsonNet()
.RegisterUnhandledExceptionHandler();
configuration.SetDefault <IChunkStatisticService, DefaultChunkStatisticService>();
var root = @"c:\estore-files";
var path = Path.Combine(root, @"stream-index-chunks");
var singleFileIndexCount = 100 * 10000;
var dataUnitSize = 68;
var indexCountForSingleAgg = 10;
var totalAggNumber = 50 * 10000;
var config = new ChunkManagerConfig(
path,
new DefaultFileNamingStrategy("stream-index-"),
0,
dataUnitSize,
singleFileIndexCount,
100,
false,
false,
Environment.ProcessorCount * 2,
12,
128 * 1024,
128 * 1024,
90,
45,
1,
5,
1000000,
false,
false);
var chunkManager = new ChunkManager("default", config);
CreateChunks(root, path, chunkManager, singleFileIndexCount, dataUnitSize, totalAggNumber, indexCountForSingleAgg);
ReadAggIndexTest(chunkManager, singleFileIndexCount, dataUnitSize, indexCountForSingleAgg);
Console.ReadLine();
}
public void Init(ChunkManagerConfig eventDataChunkConfig, ChunkManagerConfig aggregateIndexChunkConfig, ChunkManagerConfig commandIndexChunkConfig)
{
_eventDataChunkManager = new ChunkManager("EventDataChunk", eventDataChunkConfig, false);
_eventDataChunkWriter = new ChunkWriter(_eventDataChunkManager);
_eventDataChunkReader = new ChunkReader(_eventDataChunkManager, _eventDataChunkWriter);
_aggregateIndexChunkManager = new ChunkManager("AggregateIndexChunk", aggregateIndexChunkConfig, false);
_aggregateIndexChunkWriter = new ChunkWriter(_aggregateIndexChunkManager);
_aggregateIndexChunkReader = new ChunkReader(_aggregateIndexChunkManager, _aggregateIndexChunkWriter);
_commandIndexChunkManager = new ChunkManager("CommandIndexChunk", commandIndexChunkConfig, false);
_commandIndexChunkWriter = new ChunkWriter(_commandIndexChunkManager);
_commandIndexChunkReader = new ChunkReader(_commandIndexChunkManager, _commandIndexChunkWriter);
_eventDataChunkManager.Load(ReadEventStreamRecord);
_aggregateIndexChunkManager.Load(ReadIndexRecord);
_commandIndexChunkManager.Load(ReadIndexRecord);
_eventDataChunkWriter.Open();
_aggregateIndexChunkWriter.Open();
_commandIndexChunkWriter.Open();
LoadIndexData();
}
public BrokerSetting(bool isMessageStoreMemoryMode = false, string chunkFileStoreRootPath = @"c:\equeue-store", int messageChunkDataSize = 1024 * 1024 * 1024, int chunkFlushInterval = 100, int chunkCacheMaxCount = 10, int chunkCacheMinCount = 2, int maxLogRecordSize = 5 * 1024 * 1024, int chunkWriteBuffer = 128 * 1024, int chunkReadBuffer = 128 * 1024, bool syncFlush = false, FlushOption flushOption = FlushOption.FlushToOS, bool enableCache = true, int messageChunkLocalCacheSize = 300000, int queueChunkLocalCacheSize = 10000)
{
BrokerInfo = new BrokerInfo(
"DefaultBroker",
"DefaultGroup",
"DefaultCluster",
BrokerRole.Master,
new IPEndPoint(IPAddress.Loopback, 5000).ToAddress(),
new IPEndPoint(IPAddress.Loopback, 5001).ToAddress(),
new IPEndPoint(IPAddress.Loopback, 5002).ToAddress());
NameServerList = new List <IPEndPoint>()
{
new IPEndPoint(IPAddress.Loopback, 9493)
};
NotifyWhenMessageArrived = true;
RegisterBrokerToNameServerInterval = 1000 * 5;
DeleteMessagesInterval = 1000 * 10;
DeleteQueueMessagesInterval = 1000 * 10;
DeleteMessageIgnoreUnConsumed = true;
PersistConsumeOffsetInterval = 1000 * 1;
CheckBlockingPullRequestMilliseconds = 1000 * 1;
ProducerExpiredTimeout = 1000 * 30;
ConsumerExpiredTimeout = 1000 * 30;
RemoveConsumerWhenDisconnect = true;
AutoCreateTopic = true;
TopicDefaultQueueCount = 4;
TopicMaxQueueCount = 256;
MessageMaxSize = 1024 * 1024 * 4;
MessageWriteQueueThreshold = 2 * 10000;
BatchMessageWriteQueueThreshold = 1 * 10000;
IsMessageStoreMemoryMode = isMessageStoreMemoryMode;
FileStoreRootPath = chunkFileStoreRootPath;
LatestMessageShowCount = 100;
MessageChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, @"message-chunks"),
new DefaultFileNamingStrategy("message-chunk-"),
messageChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
1,
5,
messageChunkLocalCacheSize,
true);
QueueChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, @"queue-chunks"),
new DefaultFileNamingStrategy("queue-chunk-"),
0,
12,
1000000,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 2,
12,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
1,
5,
queueChunkLocalCacheSize,
false);
}
static void WriteChunkPerfTest()
{
ECommonConfiguration
.Create()
.UseAutofac()
.RegisterCommonComponents()
.UseLog4Net()
.RegisterUnhandledExceptionHandler()
.BuildContainer();
var storeRootPath = ConfigurationManager.AppSettings["storeRootPath"]; //文件存储根目录
var threadCount = int.Parse(ConfigurationManager.AppSettings["concurrentThreadCount"]); //并行写数据的线程数
var recordSize = int.Parse(ConfigurationManager.AppSettings["recordSize"]); //记录大小，字节为单位
var recordCount = int.Parse(ConfigurationManager.AppSettings["recordCount"]); //总共要写入的记录数
var syncFlush = bool.Parse(ConfigurationManager.AppSettings["syncFlush"]); //是否同步刷盘
FlushOption flushOption;
Enum.TryParse(ConfigurationManager.AppSettings["flushOption"], out flushOption); //同步刷盘方式
var chunkSize = 1024 * 1024 * 1024;
var flushInterval = 100;
var maxRecordSize = 5 * 1024 * 1024;
var chunkWriteBuffer = 128 * 1024;
var chunkReadBuffer = 128 * 1024;
var chunkManagerConfig = new ChunkManagerConfig(
Path.Combine(storeRootPath, @"sample-chunks"),
new DefaultFileNamingStrategy("message-chunk-"),
chunkSize,
0,
0,
flushInterval,
false,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
5,
1,
1,
5,
10 * 10000,
true);
var chunkManager = new ChunkManager("sample-chunk", chunkManagerConfig, false);
var chunkWriter = new ChunkWriter(chunkManager);
chunkManager.Load(ReadRecord);
chunkWriter.Open();
var record = new BufferLogRecord
{
RecordBuffer = new byte[recordSize]
};
var count = 0L;
var performanceService = ObjectContainer.Resolve <IPerformanceService>();
performanceService.Initialize("WriteChunk").Start();
for (var i = 0; i < threadCount; i++)
{
Task.Factory.StartNew(() =>
{
while (true)
{
var current = Interlocked.Increment(ref count);
if (current > recordCount)
{
break;
}
var start = DateTime.Now;
chunkWriter.Write(record);
performanceService.IncrementKeyCount("default", (DateTime.Now - start).TotalMilliseconds);
}
});
}
Console.ReadLine();
chunkWriter.Close();
chunkManager.Close();
}
static void AddAggregateEventTest()
{
ECommonConfiguration
.Create()
.UseAutofac()
.RegisterCommonComponents()
.UseLog4Net()
.RegisterUnhandledExceptionHandler()
.RegisterEStoreComponents()
.BuildContainer();
var chunkFileStoreRootPath = @"d:\event-store";
var eventDataChunkDataSize = 256 * 1024 * 1024;
var indexChunkDataSize = 256 * 1024 * 1024;
var chunkFlushInterval = 100;
var chunkCacheMaxCount = 15;
var chunkCacheMinCount = 15;
var maxDataLogRecordSize = 5 * 1024 * 1024;
var maxIndexLogRecordSize = 5 * 1024 * 1024;
var chunkWriteBuffer = 128 * 1024;
var chunkReadBuffer = 128 * 1024;
var syncFlush = false;
var flushOption = FlushOption.FlushToOS;
var enableCache = true;
var preCacheChunkCount = 10;
var chunkLocalCacheSize = 300000;
var eventDataChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, "event-data-chunks"),
new DefaultFileNamingStrategy("event-data-chunk-"),
eventDataChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxDataLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
preCacheChunkCount,
5,
chunkLocalCacheSize,
true);
var eventIndexChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, "event-index-chunks"),
new DefaultFileNamingStrategy("event-index-chunk-"),
indexChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxIndexLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
preCacheChunkCount,
5,
chunkLocalCacheSize,
true);
var commandIndexChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, "command-index-chunks"),
new DefaultFileNamingStrategy("command-index-chunk-"),
indexChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxIndexLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
preCacheChunkCount,
5,
chunkLocalCacheSize,
true);
var eventStore = ObjectContainer.Resolve <IEventStore>() as DefaultEventStore;
eventStore.Init(eventDataChunkConfig, eventIndexChunkConfig, commandIndexChunkConfig);
eventStore.Start();
var eventStream = new TestEventStream
{
AggregateRootId = ObjectId.GenerateNewStringId(),
AggregateRootType = "Note",
Events = "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890",
Timestamp = DateTime.UtcNow.Ticks
};
var totalCount = 10000000;
var count = 0;
var performanceService = ObjectContainer.Resolve <IPerformanceService>();
performanceService.Initialize("WriteChunk").Start();
Task.Factory.StartNew(() =>
{
while (true)
{
try
{
var current = Interlocked.Increment(ref count);
if (current > totalCount)
{
break;
}
var start = DateTime.Now;
eventStream.Version = current;
eventStream.CommandId = ObjectId.GenerateNewStringId();
eventStream.CommandCreateTimestamp = DateTime.Now.Ticks;
eventStore.AppendEventStream(eventStream);
performanceService.IncrementKeyCount("default", (DateTime.Now - start).TotalMilliseconds);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
});
Console.ReadLine();
eventStore.Stop();
}
static DefaultEventStore InitEventStore()
{
var chunkFileStoreRootPath = @"d:\event-store";
var eventDataChunkDataSize = 1024 * 1024 * 1024;
var indexChunkDataSize = 256 * 1024 * 1024;
var chunkFlushInterval = 100;
var chunkCacheMaxCount = 1;
var chunkCacheMinCount = 1;
var maxDataLogRecordSize = 5 * 1024 * 1024;
var maxIndexLogRecordSize = 5 * 1024 * 1024;
var chunkWriteBuffer = 128 * 1024;
var chunkReadBuffer = 128 * 1024;
var syncFlush = false;
var flushOption = FlushOption.FlushToOS;
var enableCache = true;
var preCacheChunkCount = 10;
var chunkLocalCacheSize = 300000;
var eventDataChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, "event-data-chunks"),
new DefaultFileNamingStrategy("event-data-chunk-"),
eventDataChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxDataLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
preCacheChunkCount,
5,
chunkLocalCacheSize,
true);
var aggregateIndexChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, "aggregate-index-chunks"),
new DefaultFileNamingStrategy("aggregate-index-chunk-"),
indexChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxIndexLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
preCacheChunkCount,
5,
chunkLocalCacheSize,
true);
var commandIndexChunkConfig = new ChunkManagerConfig(
Path.Combine(chunkFileStoreRootPath, "command-index-chunks"),
new DefaultFileNamingStrategy("command-index-chunk-"),
indexChunkDataSize,
0,
0,
chunkFlushInterval,
enableCache,
syncFlush,
flushOption,
Environment.ProcessorCount * 8,
maxIndexLogRecordSize,
chunkWriteBuffer,
chunkReadBuffer,
chunkCacheMaxCount,
chunkCacheMinCount,
preCacheChunkCount,
5,
chunkLocalCacheSize,
true);
var eventStore = ObjectContainer.Resolve <IEventStore>() as DefaultEventStore;
eventStore.Init(eventDataChunkConfig, aggregateIndexChunkConfig, commandIndexChunkConfig);
eventStore.Start();
return(eventStore);
}