when_send_queue_size_is_larger_than_threshold_should_close_connection()
{
var mre = new ManualResetEventSlim();
var messageSize = _connectionPendingSendBytesThreshold;
var evnt = new EventRecord(0, 0, Guid.NewGuid(), Guid.NewGuid(), 0, 0, "testStream", 0, DateTime.Now,
PrepareFlags.None, "eventType", new byte[messageSize], new byte[0]);
var record = ResolvedEvent.ForUnresolvedEvent(evnt, null);
var message = new ClientMessage.ReadEventCompleted(Guid.NewGuid(), "testStream", ReadEventResult.Success,
record, StreamMetadata.Empty, false, "");
var dummyConnection = new DummyTcpConnection();
dummyConnection.SendQueueSize = ESConsts.MaxConnectionQueueSize + 1;
var tcpConnectionManager = new TcpConnectionManager(
Guid.NewGuid().ToString(), TcpServiceType.External, new ClientTcpDispatcher(),
InMemoryBus.CreateTest(), dummyConnection, InMemoryBus.CreateTest(),
new InternalAuthenticationProvider(
new Core.Helpers.IODispatcher(InMemoryBus.CreateTest(), new NoopEnvelope()), null, 1, false),
new AuthorizationGateway(new TestAuthorizationProvider()),
TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), (man, err) => { mre.Set(); },
ESConsts.UnrestrictedPendingSendBytes, ESConsts.MaxConnectionQueueSize);
tcpConnectionManager.SendMessage(message);
if (!mre.Wait(2000))
{
Assert.Fail("Timed out waiting for connection to close");
}
}
public void when_handling_trusted_write_on_external_service()
{
var package = new TcpPackage(TcpCommand.WriteEvents, TcpFlags.TrustedWrite, Guid.NewGuid(), null, null,
new byte[] { });
var dummyConnection = new DummyTcpConnection();
var tcpConnectionManager = new TcpConnectionManager(
Guid.NewGuid().ToString(), TcpServiceType.External, new ClientTcpDispatcher(),
InMemoryBus.CreateTest(), dummyConnection, InMemoryBus.CreateTest(),
new InternalAuthenticationProvider(
new Core.Helpers.IODispatcher(InMemoryBus.CreateTest(), new NoopEnvelope()),
new StubPasswordHashAlgorithm(), 1, false),
new AuthorizationGateway(new TestAuthorizationProvider()),
TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), (man, err) => { },
_connectionPendingSendBytesThreshold, _connectionQueueSizeThreshold);
tcpConnectionManager.ProcessPackage(package);
var data = dummyConnection.ReceivedData.Last();
var receivedPackage = TcpPackage.FromArraySegment(data);
Assert.AreEqual(receivedPackage.Command, TcpCommand.BadRequest, "Expected Bad Request but got {0}",
receivedPackage.Command);
}
public void setup()
{
_checkpointUnhandledBytesThreshold = 1000;
_checkpointProcessedEventsThreshold = 2000;
_checkpointAfterMs = 10000;
_timeProvider = new RealTimeProvider();
Given();
_bus = new InMemoryBus("bus");
_projectionCorrelationId = Guid.NewGuid();
_eventHandler = new TestHandler <EventReaderSubscriptionMessage.CommittedEventReceived>();
_checkpointHandler = new TestHandler <EventReaderSubscriptionMessage.CheckpointSuggested>();
_progressHandler = new TestHandler <EventReaderSubscriptionMessage.ProgressChanged>();
_subscriptionStartedHandler = new TestHandler <EventReaderSubscriptionMessage.SubscriptionStarted>();
_notAuthorizedHandler = new TestHandler <EventReaderSubscriptionMessage.NotAuthorized>();
_eofHandler = new TestHandler <EventReaderSubscriptionMessage.EofReached>();
_partitionEofHandler = new TestHandler <EventReaderSubscriptionMessage.PartitionEofReached>();
_partitionDeletedHandler = new TestHandler <EventReaderSubscriptionMessage.PartitionDeleted>();
_bus.Subscribe(_eventHandler);
_bus.Subscribe(_checkpointHandler);
_bus.Subscribe(_progressHandler);
_bus.Subscribe(_eofHandler);
_bus.Subscribe(_partitionEofHandler);
_readerStrategy = CreateCheckpointStrategy();
_subscription = CreateProjectionSubscription();
When();
}
when_send_queue_size_is_smaller_than_threshold_should_not_close_connection()
{
var mre = new ManualResetEventSlim();
var messageSize = _connectionPendingSendBytesThreshold;
var evnt = new EventRecord(0, 0, Guid.NewGuid(), Guid.NewGuid(), 0, 0, "testStream", 0, DateTime.Now,
PrepareFlags.None, "eventType", new byte[messageSize], new byte[0]);
var record = ResolvedEvent.ForUnresolvedEvent(evnt, null);
var message = new ClientMessage.ReadEventCompleted(Guid.NewGuid(), "testStream", ReadEventResult.Success,
record, StreamMetadata.Empty, false, "");
var dummyConnection = new DummyTcpConnection();
dummyConnection.SendQueueSize = ESConsts.MaxConnectionQueueSize - 1;
var tcpConnectionManager = new TcpConnectionManager(
Guid.NewGuid().ToString(), TcpServiceType.External, new ClientTcpDispatcher(),
InMemoryBus.CreateTest(), dummyConnection, InMemoryBus.CreateTest(),
new InternalAuthenticationProvider(
new Core.Helpers.IODispatcher(InMemoryBus.CreateTest(), new NoopEnvelope()), null, 1, false),
new AuthorizationGateway(new TestAuthorizationProvider()),
TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), (man, err) => { mre.Set(); },
ESConsts.UnrestrictedPendingSendBytes, ESConsts.MaxConnectionQueueSize);
tcpConnectionManager.SendMessage(message);
var data = dummyConnection.ReceivedData.Last();
var receivedPackage = TcpPackage.FromArraySegment(data);
Assert.AreEqual(receivedPackage.Command, TcpCommand.ReadEventCompleted,
"Expected ReadEventCompleted but got {0}", receivedPackage.Command);
}
public void Run_Turn_Calculation()
{
/// Initialise
var log = new EmptyLog();
var locator = new InMemoryLocator();
locator.AddResolver(typeof(IRepository <Stockpile>), () => new InMemoryRepository <Stockpile>());
var bus = new InMemoryBus(locator, log);
Bus.SetBus(bus);
Bus.RegisterHandler <AddStockpileCommandHandler>();
/// Start testing
var calculator = new TurnCalculator();
var map = new Map(8, 8);
var game = new Game(map.GetAggregateKey());
var stockpileRepository = new InMemoryRepository <Stockpile>();
Bus.Send(new AddStockpileCommand(game, Resource.Collection.Iron));
Bus.Send(new AddStockpileCommand(game, Resource.Collection.Stone));
Bus.Send(new AddStockpileCommand(game, Resource.Collection.Wood));
calculator.Calculate(game);
}
public virtual void TestFixtureSetUp()
{
_publisher = new InMemoryBus("publisher");
_tfReader = new FakeInMemoryTfReader(RecordOffset);
_tableIndex = new FakeInMemoryTableIndex <string>();
_readerPool = new ObjectPool <ITransactionFileReader>(
"ReadIndex readers pool", 5, 100,
() => _tfReader);
_indexBackend = new IndexBackend <string>(_readerPool, 100000, 100000);
var logFormat = LogFormatAbstractor.V2;
_streamIds = logFormat.StreamIds;
_streamNames = logFormat.StreamNames;
_systemStreams = logFormat.SystemStreams;
_factory = logFormat.StreamNamesProvider;
_validator = logFormat.StreamIdValidator;
var emptyStreamId = logFormat.EmptyStreamId;
_sizer = logFormat.StreamIdSizer;
_indexReader = new IndexReader <string>(_indexBackend, _tableIndex, _factory, _validator, new StreamMetadata(maxCount: 100000), 100, false);
_indexWriter = new IndexWriter <string>(_indexBackend, _indexReader, _streamIds, _streamNames, _systemStreams, emptyStreamId, _sizer);
_indexCommitter = new Core.Services.Storage.ReaderIndex.IndexCommitter <string>(_publisher, _indexBackend, _indexReader, _tableIndex, _streamNames, _systemStreams, new InMemoryCheckpoint(-1), false);
WriteEvents();
}
public Response<ILoadBalancer> Create(DownstreamReRoute reRoute, IServiceDiscoveryProvider serviceProvider)
{
var loadBalancer = new RoundRobin(async () => await serviceProvider.Get());
var bus = new InMemoryBus<StickySession>();
return new OkResponse<ILoadBalancer>(new CookieStickySessions(loadBalancer, reRoute.LoadBalancerOptions.Key,
reRoute.LoadBalancerOptions.ExpiryInMs, bus));
}
public void SetupMessaging(InMemoryBus coreInputBus, InMemoryBus readerInputBus)
{
coreInputBus.Subscribe <ProjectionMessage.CoreService.Start>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.CoreService.Stop>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.CoreService.Tick>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.Projections.SubscribeProjection>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.Projections.UnsubscribeProjection>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.Projections.PauseProjectionSubscription>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.Projections.ResumeProjectionSubscription>(_projectionCoreService);
coreInputBus.Subscribe <ProjectionMessage.Projections.CommittedEventReceived>(_projectionCoreService);
coreInputBus.Subscribe <SystemMessage.SystemInit>(_projectionManager);
coreInputBus.Subscribe <SystemMessage.SystemStart>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.Post>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.UpdateQuery>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.GetQuery>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.Delete>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.GetStatistics>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.GetState>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.Disable>(_projectionManager);
coreInputBus.Subscribe <ProjectionManagementMessage.Enable>(_projectionManager);
coreInputBus.Subscribe <ProjectionMessage.Projections.Started>(_projectionManager);
coreInputBus.Subscribe <ProjectionMessage.Projections.Stopped>(_projectionManager);
coreInputBus.Subscribe <ProjectionMessage.Projections.Faulted>(_projectionManager);
coreInputBus.Subscribe <ClientMessage.WriteEventsCompleted>(_projectionManager);
coreInputBus.Subscribe <ClientMessage.ReadEventsBackwardsCompleted>(_projectionManager);
//NOTE: message forwarding is set up outside (for Read/Write events)
coreInputBus.Subscribe(this);
}
public ProjectionsSubsystem(ProjectionSubsystemOptions projectionSubsystemOptions)
{
if (projectionSubsystemOptions.RunProjections <= ProjectionType.System)
{
_projectionWorkerThreadCount = 1;
}
else
{
_projectionWorkerThreadCount = projectionSubsystemOptions.ProjectionWorkerThreadCount;
}
_runProjections = projectionSubsystemOptions.RunProjections;
// Projection manager & Projection Core Coordinator
// The manager only starts when projections are running
_componentCount = _runProjections == ProjectionType.None ? 1 : 2;
// Projection manager & each projection core worker
_dispatcherCount = 1 + _projectionWorkerThreadCount;
_startStandardProjections = projectionSubsystemOptions.StartStandardProjections;
_projectionsQueryExpiry = projectionSubsystemOptions.ProjectionQueryExpiry;
_faultOutOfOrderProjections = projectionSubsystemOptions.FaultOutOfOrderProjections;
_leaderMainBus = new InMemoryBus("manager input bus");
_subsystemInitialized = 0;
_projectionRuntime = projectionSubsystemOptions.Runtime;
_executionTimeout = projectionSubsystemOptions.ExecutionTimeout;
_compilationTimeout = projectionSubsystemOptions.CompilationTimeout;
}
public void of_http_requests_routing()
{
const int iterations = 100000;
IPublisher inputBus = new NoopPublisher();
var bus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(bus, "Test", true, TimeSpan.FromMilliseconds(50));
var multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
var providers = new HttpAuthenticationProvider[] { new AnonymousHttpAuthenticationProvider() };
var httpService = new HttpService(ServiceAccessibility.Public, inputBus,
new TrieUriRouter(), multiQueuedHandler, "http://localhost:12345/");
HttpService.CreateAndSubscribePipeline(bus, providers);
var fakeController = new FakeController(iterations, null);
httpService.SetupController(fakeController);
httpService.Handle(new SystemMessage.SystemInit());
var rnd = new Random();
var sw = Stopwatch.StartNew();
var httpClient = new HttpAsyncClient();
for (int i = 0; i < iterations; ++i)
{
var route = fakeController.BoundRoutes[rnd.Next(0, fakeController.BoundRoutes.Count)];
switch (route.Item2)
{
case HttpMethod.Get:
httpClient.Get(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Post:
httpClient.Post(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Delete:
httpClient.Delete(route.Item1, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
case HttpMethod.Put:
httpClient.Put(route.Item1, "abracadabra", ContentType.Json, TimeSpan.FromMilliseconds(10000), x => { }, x => { throw new Exception(); });
break;
default:
throw new Exception();
}
}
fakeController.CountdownEvent.Wait();
sw.Stop();
Console.WriteLine("{0} request done in {1} ({2:0.00} per sec)", iterations, sw.Elapsed, 1000.0 * iterations / sw.ElapsedMilliseconds);
httpService.Shutdown();
multiQueuedHandler.Stop();
}
public void Setup()
{
_consumer = new TestHandler <Message>();
_bus = new InMemoryBus("temp");
_bus.Subscribe(_consumer);
ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000);
var ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
_readerService = new EventReaderCoreService(_bus, ioDispatcher, 10, writerCheckpoint, runHeadingReader: true);
_subscriptionDispatcher =
new ReaderSubscriptionDispatcher(_bus);
_spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(_bus);
_timeoutScheduler = new TimeoutScheduler();
_workerId = Guid.NewGuid();
_service = new ProjectionCoreService(
_workerId, _bus, _bus, _subscriptionDispatcher, new RealTimeProvider(), ioDispatcher,
_spoolProcessingResponseDispatcher, _timeoutScheduler);
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionEofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionMeasured>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.PartitionDeleted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.SubscriptionStarted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber <EventReaderSubscriptionMessage.ReaderAssignedReader>());
_bus.Subscribe(_spoolProcessingResponseDispatcher.CreateSubscriber <PartitionProcessingResult>());
_readerService.Handle(new Messages.ReaderCoreServiceMessage.StartReader());
_service.Handle(new ProjectionCoreServiceMessage.StartCore());
}
public void Setup()
{
_consumer = new TestHandler<Message>();
_bus = new InMemoryBus("temp");
_bus.Subscribe(_consumer);
ICheckpoint writerCheckpoint = new InMemoryCheckpoint(1000);
var ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
_readerService = new EventReaderCoreService(_bus, ioDispatcher, 10, writerCheckpoint, runHeadingReader: true);
_subscriptionDispatcher =
new ReaderSubscriptionDispatcher(_bus);
_spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(_bus);
_timeoutScheduler = new TimeoutScheduler();
_workerId = Guid.NewGuid();
_service = new ProjectionCoreService(
_workerId, _bus, _bus, _subscriptionDispatcher, new RealTimeProvider(), ioDispatcher,
_spoolProcessingResponseDispatcher, _timeoutScheduler);
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionEofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionMeasured>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionDeleted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.SubscriptionStarted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ReaderAssignedReader>());
_bus.Subscribe(_spoolProcessingResponseDispatcher.CreateSubscriber<PartitionProcessingResult>());
_readerService.Handle(new Messages.ReaderCoreServiceMessage.StartReader());
_service.Handle(new ProjectionCoreServiceMessage.StartCore());
}
public ElectionsServiceUnit(ClusterSettings clusterSettings)
{
Publisher = new FakePublisher();
_bus = new InMemoryBus(GetType().Name);
var memberInfo = MemberInfo.Initial(clusterSettings.Self.NodeInfo.InstanceId, InitialDate,
VNodeState.Unknown, true,
clusterSettings.Self.NodeInfo.InternalTcp,
clusterSettings.Self.NodeInfo.InternalSecureTcp,
clusterSettings.Self.NodeInfo.ExternalTcp,
clusterSettings.Self.NodeInfo.ExternalSecureTcp,
clusterSettings.Self.NodeInfo.HttpEndPoint, null, 0, 0,
clusterSettings.Self.NodePriority,
clusterSettings.Self.ReadOnlyReplica);
ElectionsService = new Core.Services.ElectionsService(Publisher,
memberInfo,
clusterSettings.ClusterNodesCount,
new InMemoryCheckpoint(WriterCheckpoint),
new InMemoryCheckpoint(ChaserCheckpoint),
new FakeEpochManager(),
() => - 1, 0, new FakeTimeProvider());
ElectionsService.SubscribeMessages(_bus);
InputMessages = new List <Message>();
InitialClusterSettings = clusterSettings;
InitialClusterInfo = BuildClusterInfo(clusterSettings);
ClusterInfo = new ClusterInfo(InitialClusterInfo.Members);
}
public void setup()
{
_checkpointUnhandledBytesThreshold = 1000;
_checkpointProcessedEventsThreshold = 2000;
Given();
_bus = new InMemoryBus("bus");
_projectionCorrelationId = Guid.NewGuid();
_eventHandler = new TestHandler<EventReaderSubscriptionMessage.CommittedEventReceived>();
_checkpointHandler = new TestHandler<EventReaderSubscriptionMessage.CheckpointSuggested>();
_progressHandler = new TestHandler<EventReaderSubscriptionMessage.ProgressChanged>();
_subscriptionStartedHandler = new TestHandler<EventReaderSubscriptionMessage.SubscriptionStarted>();
_notAuthorizedHandler = new TestHandler<EventReaderSubscriptionMessage.NotAuthorized>();
_eofHandler = new TestHandler<EventReaderSubscriptionMessage.EofReached>();
_partitionEofHandler = new TestHandler<EventReaderSubscriptionMessage.PartitionEofReached>();
_partitionMeasuredHandler = new TestHandler<EventReaderSubscriptionMessage.PartitionMeasured>();
_bus.Subscribe(_eventHandler);
_bus.Subscribe(_checkpointHandler);
_bus.Subscribe(_progressHandler);
_bus.Subscribe(_eofHandler);
_bus.Subscribe(_partitionEofHandler);
_bus.Subscribe(_partitionMeasuredHandler);
_readerStrategy = CreateCheckpointStrategy();
_subscription = CreateProjectionSubscription();
When();
}
private Guid AddSubscription(Guid replicaId, bool isPromotable, out TcpConnectionManager manager)
{
var tcpConn = new DummyTcpConnection()
{
ConnectionId = replicaId
};
manager = new TcpConnectionManager(
"Test Subscription Connection manager", TcpServiceType.External, new ClientTcpDispatcher(),
InMemoryBus.CreateTest(), tcpConn, InMemoryBus.CreateTest(),
new InternalAuthenticationProvider(
new Core.Helpers.IODispatcher(InMemoryBus.CreateTest(), new NoopEnvelope()),
new StubPasswordHashAlgorithm(), 1, false),
TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), (man, err) => { },
_connectionPendingSendBytesThreshold, _connectionQueueSizeThreshold);
var subRequest = new ReplicationMessage.ReplicaSubscriptionRequest(
Guid.NewGuid(),
new NoopEnvelope(),
manager,
0,
Guid.NewGuid(),
new Epoch[0],
PortsHelper.GetLoopback(),
MasterId,
replicaId,
isPromotable);
Service.Handle(subRequest);
return(tcpConn.ConnectionId);
}
public void Register(StandardComponents standardComponents)
{
_leaderInputQueue = QueuedHandler.CreateQueuedHandler(_leaderMainBus, "Projections Leader",
standardComponents.QueueStatsManager);
_leaderOutputBus = new InMemoryBus("ProjectionManagerAndCoreCoordinatorOutput");
_leaderMainBus.Subscribe <ProjectionSubsystemMessage.RestartSubsystem>(this);
_leaderMainBus.Subscribe <ProjectionSubsystemMessage.ComponentStarted>(this);
_leaderMainBus.Subscribe <ProjectionSubsystemMessage.ComponentStopped>(this);
_leaderMainBus.Subscribe <ProjectionSubsystemMessage.IODispatcherDrained>(this);
_leaderMainBus.Subscribe <SystemMessage.SystemCoreReady>(this);
_leaderMainBus.Subscribe <SystemMessage.StateChangeMessage>(this);
var projectionsStandardComponents = new ProjectionsStandardComponents(
_projectionWorkerThreadCount,
_runProjections,
_leaderOutputBus,
_leaderInputQueue,
_leaderMainBus,
_faultOutOfOrderProjections,
_projectionRuntime, _compilationTimeout, _executionTimeout);
CreateAwakerService(standardComponents);
_coreQueues =
ProjectionCoreWorkersNode.CreateCoreWorkers(standardComponents, projectionsStandardComponents);
_queueMap = _coreQueues.ToDictionary(v => v.Key, v => (IPublisher)v.Value);
ProjectionManagerNode.CreateManagerService(standardComponents, projectionsStandardComponents, _queueMap,
_projectionsQueryExpiry);
projectionsStandardComponents.LeaderMainBus.Subscribe <CoreProjectionStatusMessage.Stopped>(this);
projectionsStandardComponents.LeaderMainBus.Subscribe <CoreProjectionStatusMessage.Started>(this);
}
private void SetupMessaging(ISubscriber subscriber)
{
var storageReaderBus = new InMemoryBus("StorageReaderBus");
storageReaderBus.Subscribe <SystemMessage.SystemInit>(this);
storageReaderBus.Subscribe <SystemMessage.BecomeShuttingDown>(this);
storageReaderBus.Subscribe <ClientMessage.ReadEvent>(this);
storageReaderBus.Subscribe <ClientMessage.ReadStreamEventsBackward>(this);
storageReaderBus.Subscribe <ClientMessage.ReadStreamEventsForward>(this);
storageReaderBus.Subscribe <ClientMessage.ReadAllEventsForward>(this);
storageReaderBus.Subscribe <ClientMessage.ReadAllEventsBackward>(this);
storageReaderBus.Subscribe <ClientMessage.ListStreams>(this);
subscriber.Subscribe(this.WidenFrom <SystemMessage.SystemInit, Message>());
subscriber.Subscribe(this.WidenFrom <SystemMessage.BecomeShuttingDown, Message>());
subscriber.Subscribe(this.WidenFrom <ClientMessage.ReadEvent, Message>());
subscriber.Subscribe(this.WidenFrom <ClientMessage.ReadStreamEventsBackward, Message>());
subscriber.Subscribe(this.WidenFrom <ClientMessage.ReadStreamEventsForward, Message>());
subscriber.Subscribe(this.WidenFrom <ClientMessage.ReadAllEventsForward, Message>());
subscriber.Subscribe(this.WidenFrom <ClientMessage.ReadAllEventsBackward, Message>());
subscriber.Subscribe(this.WidenFrom <ClientMessage.ListStreams, Message>());
_storageReaderQueues = new QueuedHandler[_threadCount];
for (int i = 0; i < _threadCount; ++i)
{
var queue = new QueuedHandler(storageReaderBus, string.Format("StorageReaderQueue #{0}", i));
_storageReaderQueues[i] = queue;
queue.Start();
}
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestHandler<ClientMessage.ReadStreamEventsBackward>();
_bus.Subscribe(_listEventsHandler);
_ioDispatcher = new IODispatcher(_bus, new PublishEnvelope(_bus));
_subscriptionDispatcher =
new ReaderSubscriptionDispatcher
(_bus);
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionEofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.PartitionMeasured>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.SubscriptionStarted>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ReaderAssignedReader>());
_bus.Subscribe(_ioDispatcher.BackwardReader);
_bus.Subscribe(_ioDispatcher.ForwardReader);
_bus.Subscribe(_ioDispatcher.Writer);
_bus.Subscribe(_ioDispatcher);
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
_projectionConfig = new ProjectionConfig(null, 5, 10, 1000, 250, true, true, false, false, false);
var version = new ProjectionVersion(1, 0, 0);
var projectionProcessingStrategy = new ContinuousProjectionProcessingStrategy(
"projection", version, projectionStateHandler, _projectionConfig,
projectionStateHandler.GetSourceDefinition(), null, _subscriptionDispatcher);
_coreProjection = projectionProcessingStrategy.Create(
Guid.NewGuid(), _bus, SystemAccount.Principal, _bus, _ioDispatcher, _subscriptionDispatcher,
new RealTimeProvider());
_coreProjection.Start();
}
public void SetUp(Action <IHttpService> bootstrap = null)
{
_bus = new InMemoryBus($"bus_{_serverEndPoint.Port}");
var pipelineBus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", new QueueStatsManager(), true, TimeSpan.FromMilliseconds(50));
_multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[] { queue }, null);
_multiQueuedHandler.Start();
_service = new KestrelHttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
_multiQueuedHandler, false, null, 0, false, _serverEndPoint);
_internalDispatcher = new InternalDispatcherEndpoint(queue, _multiQueuedHandler);
_bus.Subscribe(_internalDispatcher);
KestrelHttpService.CreateAndSubscribePipeline(pipelineBus);
bootstrap?.Invoke(_service);
_server = new TestServer(
new WebHostBuilder()
.UseStartup(new ClusterVNodeStartup <string>(Array.Empty <ISubsystem>(), queue, queue, _bus, _multiQueuedHandler,
new TestAuthenticationProvider(),
new IHttpAuthenticationProvider[] {
new BasicHttpAuthenticationProvider(new TestAuthenticationProvider()),
new AnonymousHttpAuthenticationProvider(),
}, new TestAuthorizationProvider(),
new FakeReadIndex <LogFormat.V2, string>(_ => false, new LogV2SystemStreams()),
1024 * 1024, _timeout, _service)));
_httpMessageHandler = _server.CreateHandler();
_client = new HttpAsyncClient(_timeout, _httpMessageHandler);
HttpBootstrap.Subscribe(_bus, _service);
}
when_limit_pending_and_sending_message_larger_than_pending_bytes_threshold_but_no_bytes_pending_should_not_close_connection()
{
var messageSize = _connectionPendingSendBytesThreshold + 1000;
var evnt = new EventRecord(0, 0, Guid.NewGuid(), Guid.NewGuid(), 0, 0, "testStream", 0, DateTime.Now,
PrepareFlags.None, "eventType", new byte[messageSize], new byte[0]);
var record = ResolvedEvent.ForUnresolvedEvent(evnt, null);
var message = new ClientMessage.ReadEventCompleted(Guid.NewGuid(), "testStream", ReadEventResult.Success,
record, StreamMetadata.Empty, false, "");
var dummyConnection = new DummyTcpConnection();
dummyConnection.PendingSendBytes = 0;
var tcpConnectionManager = new TcpConnectionManager(
Guid.NewGuid().ToString(), TcpServiceType.External, new ClientTcpDispatcher(),
InMemoryBus.CreateTest(), dummyConnection, InMemoryBus.CreateTest(),
new InternalAuthenticationProvider(
new Core.Helpers.IODispatcher(InMemoryBus.CreateTest(), new NoopEnvelope()), null, 1),
TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10), (man, err) => { },
_connectionPendingSendBytesThreshold);
tcpConnectionManager.SendMessage(message);
var data = dummyConnection.ReceivedData.Last();
var receivedPackage = TcpPackage.FromArraySegment(data);
Assert.AreEqual(receivedPackage.Command, TcpCommand.ReadEventCompleted,
"Expected ReadEventCompleted but got {0}", receivedPackage.Command);
}
static void Main(string[] args)
{
var outputBus = new InMemoryBus("OutputBus");
var controller = new NodeController(outputBus);
var mainQueue = new QueuedHandler(controller, "Main Queue");
controller.SetMainQueue(mainQueue);
// Hello world service
var hello = new HelloWorldService(mainQueue);
outputBus.Subscribe<SystemMessage.SystemInit>(hello);
outputBus.Subscribe<SystemMessage.StartShutdown>(hello);
outputBus.Subscribe<HelloWorldMessage.Hi>(hello);
// TIMER
var timer = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
outputBus.Subscribe<TimerMessage.Schedule>(timer);
Console.WriteLine("Starting everything. Press enter to initiate shutdown");
mainQueue.Start();
mainQueue.Publish(new SystemMessage.SystemInit());
Console.ReadLine();
mainQueue.Publish(new SystemMessage.StartShutdown());
Console.ReadLine();
}
public IAuthenticationProvider BuildAuthenticationProvider(IPublisher mainQueue, IBus mainBus, IPublisher workersQueue, InMemoryBus[] workerBusses)
{
var passwordHashAlgorithm = new Rfc2898PasswordHashAlgorithm();
var dispatcher = new IODispatcher(mainQueue, new PublishEnvelope(workersQueue, crossThread: true));
foreach (var bus in workerBusses) {
bus.Subscribe(dispatcher.ForwardReader);
bus.Subscribe(dispatcher.BackwardReader);
bus.Subscribe(dispatcher.Writer);
bus.Subscribe(dispatcher.StreamDeleter);
bus.Subscribe(dispatcher);
}
// USER MANAGEMENT
var ioDispatcher = new IODispatcher(mainQueue, new PublishEnvelope(mainQueue));
mainBus.Subscribe(ioDispatcher.BackwardReader);
mainBus.Subscribe(ioDispatcher.ForwardReader);
mainBus.Subscribe(ioDispatcher.Writer);
mainBus.Subscribe(ioDispatcher.StreamDeleter);
mainBus.Subscribe(ioDispatcher);
var userManagement = new UserManagementService(mainQueue, ioDispatcher, passwordHashAlgorithm, skipInitializeStandardUsersCheck: false);
mainBus.Subscribe<UserManagementMessage.Create>(userManagement);
mainBus.Subscribe<UserManagementMessage.Update>(userManagement);
mainBus.Subscribe<UserManagementMessage.Enable>(userManagement);
mainBus.Subscribe<UserManagementMessage.Disable>(userManagement);
mainBus.Subscribe<UserManagementMessage.Delete>(userManagement);
mainBus.Subscribe<UserManagementMessage.ResetPassword>(userManagement);
mainBus.Subscribe<UserManagementMessage.ChangePassword>(userManagement);
mainBus.Subscribe<UserManagementMessage.Get>(userManagement);
mainBus.Subscribe<UserManagementMessage.GetAll>(userManagement);
mainBus.Subscribe<SystemMessage.BecomeMaster>(userManagement);
return new InternalAuthenticationProvider(dispatcher, passwordHashAlgorithm, ESConsts.CachedPrincipalCount);
}
public virtual void Init()
{
var sendOverHttpHandler = GetSendOverHttpProcessor();
for (int i = 0; i < InstancesCnt; ++i)
{
var inputBus = new InMemoryBus(string.Format("ELECTIONS-INPUT-BUS-{0}", i));
var outputBus = new InMemoryBus(string.Format("ELECTIONS-OUTPUT-BUS-{0}", i));
var endPoint = new IPEndPoint(BaseEndPoint.Address, BaseEndPoint.Port + i);
var nodeInfo = new VNodeInfo(Guid.NewGuid(), endPoint, endPoint, endPoint, endPoint, endPoint, endPoint);
_instances.Add(new ElectionsInstance(nodeInfo.InstanceId, endPoint, inputBus, outputBus));
sendOverHttpHandler.RegisterEndPoint(endPoint, inputBus);
var electionsService = new Core.Services.ElectionsService(outputBus,
nodeInfo,
InstancesCnt,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new FakeEpochManager(),
() => -1, 0);
electionsService.SubscribeMessages(inputBus);
outputBus.Subscribe(sendOverHttpHandler);
outputBus.Subscribe(new TimerMessageProcessor(Rnd,
Runner,
endPoint,
inputBus,
_timerMinDelay,
_timerMaxDelay));
outputBus.Subscribe(new InnerBusMessagesProcessor(Runner, endPoint, inputBus));
}
}
private ProjectionManagerNode(IPublisher inputQueue, IPublisher[] queues, RunProjections runProjections)
{
_runProjections = runProjections;
_output = new InMemoryBus("ProjectionManagerOutput");
_projectionManager = new ProjectionManager(
inputQueue, _output, queues, new RealTimeProvider(), runProjections);
}
public Projections(
TFChunkDb db, QueuedHandler mainQueue, InMemoryBus mainBus, TimerService timerService,
HttpService httpService, int projectionWorkerThreadCount)
{
_projectionWorkerThreadCount = projectionWorkerThreadCount;
SetupMessaging(db, mainQueue, mainBus, timerService, httpService);
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestHandler<ClientMessage.ReadStreamEventsBackward>();
_bus.Subscribe(_listEventsHandler);
_readDispatcher = new RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_writeDispatcher = new RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_subscriptionDispatcher =
new PublishSubscribeDispatcher
<ReaderSubscriptionManagement.Subscribe,
ReaderSubscriptionManagement.ReaderSubscriptionManagementMessage, EventReaderSubscriptionMessage>
(_bus, v => v.SubscriptionId, v => v.SubscriptionId);
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CommittedEventReceived>());
_bus.Subscribe(
_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.CheckpointSuggested>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.EofReached>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.ProgressChanged>());
_bus.Subscribe(_subscriptionDispatcher.CreateSubscriber<EventReaderSubscriptionMessage.NotAuthorized>());
_bus.Subscribe(_readDispatcher);
_bus.Subscribe(_writeDispatcher);
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
_projectionConfig = new ProjectionConfig(null, 5, 10, 1000, 250, true, true, false, false);
_coreProjection = CoreProjection.CreateAndPrepare(
"projection", new ProjectionVersion(1, 0, 0), Guid.NewGuid(), _bus, projectionStateHandler, _projectionConfig, _readDispatcher,
_writeDispatcher, _subscriptionDispatcher, null, new RealTimeProvider());
_coreProjection.Start();
}
public void RegisterSaga_should_not_throw_InvalidOperationException_on_type_arguments_that_implement_IAmStartedBy_interface()
{
var containerMock = new Mock <ITypeResolver>().Object;
IBus bus = new InMemoryBus(containerMock);
bus.RegisterSaga <DummySaga>();
}
private VNodeFSM CreateFSM()
{
var outputBus = new InMemoryBus("a", false);
var state = VNodeState.Master;
var stm = new VNodeFSMBuilder(() => state)
.InAnyState()
.When <SystemMessage.StateChangeMessage>().Do(m => { })
.InState(VNodeState.Initializing)
.When <SystemMessage.SystemInit>().Do(msg => { })
.When <SystemMessage.SystemStart>().Do(msg => { })
.When <SystemMessage.BecomePreMaster>().Do(msg => { })
.When <SystemMessage.ServiceInitialized>().Do(msg => { })
.WhenOther().ForwardTo(outputBus)
.InStates(VNodeState.Initializing, VNodeState.ShuttingDown, VNodeState.Shutdown)
.When <ClientMessage.ReadRequestMessage>().Do(msg => { })
.InAllStatesExcept(VNodeState.Initializing, VNodeState.ShuttingDown, VNodeState.Shutdown)
.When <ClientMessage.ReadRequestMessage>().ForwardTo(outputBus)
.InAllStatesExcept(VNodeState.PreMaster)
.When <SystemMessage.WaitForChaserToCatchUp>().Ignore()
.When <SystemMessage.ChaserCaughtUp>().Ignore()
.InState(VNodeState.PreMaster)
.When <SystemMessage.BecomeMaster>().Do(msg => { })
.When <SystemMessage.WaitForChaserToCatchUp>().ForwardTo(outputBus)
.When <SystemMessage.ChaserCaughtUp>().Do(msg => { })
.WhenOther().ForwardTo(outputBus)
.InState(VNodeState.Master)
.When <ClientMessage.WriteEvents>().Do(msg => { })
.When <ClientMessage.TransactionStart>().Do(msg => { })
.When <ClientMessage.TransactionWrite>().Do(msg => { })
.When <ClientMessage.TransactionCommit>().Do(msg => { })
.When <ClientMessage.DeleteStream>().Do(msg => { })
.When <StorageMessage.WritePrepares>().ForwardTo(outputBus)
.When <StorageMessage.WriteDelete>().ForwardTo(outputBus)
.When <StorageMessage.WriteTransactionStart>().ForwardTo(outputBus)
.When <StorageMessage.WriteTransactionData>().ForwardTo(outputBus)
.When <StorageMessage.WriteTransactionPrepare>().ForwardTo(outputBus)
.When <StorageMessage.WriteCommit>().ForwardTo(outputBus)
.WhenOther().ForwardTo(outputBus)
.InAllStatesExcept(VNodeState.Master)
.When <ClientMessage.WriteRequestMessage>().Do(msg => { })
.When <StorageMessage.WritePrepares>().Ignore()
.When <StorageMessage.WriteDelete>().Ignore()
.When <StorageMessage.WriteTransactionStart>().Ignore()
.When <StorageMessage.WriteTransactionData>().Ignore()
.When <StorageMessage.WriteTransactionPrepare>().Ignore()
.When <StorageMessage.WriteCommit>().Ignore()
.InAllStatesExcept(VNodeState.ShuttingDown, VNodeState.Shutdown)
.When <ClientMessage.RequestShutdown>().Do(msg => { })
.When <SystemMessage.BecomeShuttingDown>().Do(msg => { })
.InState(VNodeState.ShuttingDown)
.When <SystemMessage.BecomeShutdown>().Do(msg => { })
.When <SystemMessage.ShutdownTimeout>().Do(msg => { })
.InStates(VNodeState.ShuttingDown, VNodeState.Shutdown)
.When <SystemMessage.ServiceShutdown>().Do(msg => { })
.WhenOther().ForwardTo(outputBus)
.Build();
return(stm);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
Db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
10000,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
PTableVersions.Index64Bit,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: true,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
_scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
private void DisposableTarget(InMemoryBus bus, Action action)
{
using (var target = new OneBlankMessageDisposableClass(action))
{
bus.AutoWire(target);
bus.Publish(new BlankMessage());
}
}
public void Setup()
{
_consumer = new WatchingConsumer();
_bus = new InMemoryBus("temp");
_bus.Subscribe(_consumer);
_service = new ProjectionCoreService(_bus, _bus, 10, new InMemoryCheckpoint(1000));
_service.Handle(new ProjectionMessage.CoreService.Start());
}
public ProjectionWorkerNode(TFChunkDb db, QueuedHandler inputQueue)
{
Ensure.NotNull(db, "db");
_coreOutput = new InMemoryBus("Core Output");
_projectionCoreService = new ProjectionCoreService(CoreOutput, inputQueue, 10, db.Config.WriterCheckpoint);
}
public void Setup()
{
_consumer = new WatchingConsumer();
_bus = new InMemoryBus("temp");
_bus.Subscribe(_consumer);
_service = new ProjectionCoreService(_bus, _bus, 10, new InMemoryCheckpoint(1000));
_service.Handle(new ProjectionMessage.CoreService.Start());
}
public ProjectionWorkerNode(TFChunkDb db, QueuedHandler inputQueue)
{
Ensure.NotNull(db, "db");
_coreOutput = new InMemoryBus("Core Output");
_projectionCoreService = new ProjectionCoreService(CoreOutput, inputQueue, 10, db.Config.WriterCheckpoint);
}
public void TestName()
{
ICommandDispatcher cmdDispatcher = container.GetInstance <ICommandDispatcher>();
IEventPublisher eventPublisher = container.GetInstance <IEventPublisher>();
IMessageBus bus = new InMemoryBus(cmdDispatcher, eventPublisher);
bus.Send(new TheCommand("My Command"));
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
WriterCheckpoint = new InMemoryCheckpoint(0);
ChaserCheckpoint = new InMemoryCheckpoint(0);
ReplicationCheckpoint = new InMemoryCheckpoint(-1);
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
Db = new TFChunkDb(TFChunkHelper.CreateDbConfig(PathName, WriterCheckpoint, ChaserCheckpoint, replicationCheckpoint: ReplicationCheckpoint));
Db.Open();
// create db
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
var readers = new ObjectPool <ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
var lowHasher = new XXHashUnsafe();
var highHasher = new Murmur3AUnsafe();
TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
() => new HashListMemTable(IndexBitnessVersion, MaxEntriesInMemTable * 2),
() => new TFReaderLease(readers),
IndexBitnessVersion,
MaxEntriesInMemTable);
ReadIndex = new ReadIndex(new NoopPublisher(),
readers,
TableIndex,
0,
additionalCommitChecks: PerformAdditionalCommitChecks,
metastreamMaxCount: MetastreamMaxCount,
hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault,
skipIndexScanOnReads: Opts.SkipIndexScanOnReadsDefault,
replicationCheckpoint: Db.Config.ReplicationCheckpoint);
ReadIndex.Init(ChaserCheckpoint.Read());
// scavenge must run after readIndex is built
if (_scavenge)
{
if (_completeLastChunkOnScavenge)
{
Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
}
_scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
_scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
}
}
public void RegisterSaga_should_throw_InvalidOperationException_sagas_that_implements_IAmStartedBy_more_than_once()
{
var containerMock = new Mock <ITypeResolver>().Object;
IBus bus = new InMemoryBus(containerMock);
Executing.This(() => bus.RegisterSaga <OverloadedSaga>())
.Should()
.Throw <InvalidOperationException>();
}
public void SetUp()
{
_bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));
_service = new HttpService(ServiceAccessibility.Private, _bus, 1, _serverEndPoint.ToHttpUrl());
_client = new HttpAsyncClient();
HttpBootstrap.Subscribe(_bus, _service);
}
public void When_a_constraint_doesnt_allow_any_handlers_to_be_wired()
{
var bus = new InMemoryBus();
Action <String> action = s => { };
bus.Constraints.Add(new ThrowingBeforeRegisterConstraint());
Assert.Throws <ConstraintException>(() => bus.Wire(action));
}
protected ReadModelBase(string name, Func <IListener> getListener)
{
Ensure.NotNull(getListener, nameof(getListener));
_getListener = getListener;
_listeners = new List <IListener>();
_bus = new InMemoryBus($"{nameof(ReadModelBase)}:{name} bus", false);
_queue = new QueuedHandler(_bus, $"{nameof(ReadModelBase)}:{name} queue");
_queue.Start();
}
public void SetUp()
{
_bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));
_service = new HttpService(_bus, new[] { _serverEndPoint.ToHttpUrl() });
_client = new HttpAsyncClient();
HttpBootstrap.Subscribe(_bus, _service);
}
public void When_a_constraint_doesnt_allow_publishing()
{
var bus = new InMemoryBus();
Action <String> action = s => { };
bus.Constraints.Add(new ThrowingBeforePublishConstraint());
Assert.Throws <ConstraintException>(() => bus.Publish("string"));
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
string dbPath = Path.Combine(PathName, string.Format("mini-node-db-{0}", Guid.NewGuid()));
Bus = new InMemoryBus("bus");
IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
var writerCheckFilename = Path.Combine(dbPath, Checkpoint.Writer + ".chk");
var chaserCheckFilename = Path.Combine(dbPath, Checkpoint.Chaser + ".chk");
if (Runtime.IsMono)
{
WriterCheckpoint = new FileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
ChaserCheckpoint = new FileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
}
else
{
WriterCheckpoint = new MemoryMappedFileCheckpoint(writerCheckFilename, Checkpoint.Writer, cached: true);
ChaserCheckpoint = new MemoryMappedFileCheckpoint(chaserCheckFilename, Checkpoint.Chaser, cached: true);
}
Db = new TFChunkDb(new TFChunkDbConfig(dbPath,
new VersionedPatternFileNamingStrategy(dbPath, "chunk-"),
TFConsts.ChunkSize,
0,
WriterCheckpoint,
ChaserCheckpoint,
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1),
inMemDb: false));
Db.Open();
// create DB
Writer = new TFChunkWriter(Db);
Writer.Open();
WriteTestScenario();
Writer.Close();
Writer = null;
WriterCheckpoint.Flush();
ChaserCheckpoint.Write(WriterCheckpoint.Read());
ChaserCheckpoint.Flush();
Db.Close();
// start node with our created DB
Node = new MiniNode(PathName, inMemDb: false, dbPath: dbPath);
Node.Start();
Given();
}
public void should_unsubscrible() {
var bus = new InMemoryBus();
bool subscriber_called = false;
var handler =
Handlers.Runnable<MyMessage>(msg => { subscriber_called = true; });
bus.Subscribe(handler);
bus.Unsubscribe(handler);
bus.Publish(new MyMessage {Message = "MyMessage"});
Assert.That(subscriber_called, Is.EqualTo(false));
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestMessageHandler<ClientMessage.ReadEventsBackwards>();
_bus.Subscribe(_listEventsHandler);
_coreProjection = new CoreProjection(
"projection", Guid.NewGuid(), _bus, new FakeProjectionStateHandler(),
new ProjectionConfig(ProjectionMode.AdHoc, 5, 10, 1000, 250, true, true, true));
_coreProjection.Start();
}
public void should_publish_message_to_subscribers() {
var bus = new InMemoryBus();
bool subscriber_called = false;
bus.Subscribe(Handlers.Runnable<MyMessage>(msg => {
Assert.That(msg.Message, Is.EqualTo("MyMessage"));
subscriber_called = true;
}));
bus.Publish(new MyMessage {Message = "MyMessage"});
Assert.That(subscriber_called, Is.EqualTo(true));
}
public void it_can_be_created()
{
var coreProjection = new FakeCoreProjection();
var stateCache = new PartitionStateCache();
var bus = new InMemoryBus("test");
var fakeCheckpointManager = new specification_with_multi_phase_core_projection.FakeCheckpointManager(bus, Guid.NewGuid());
var it = new WriteQueryResultProjectionProcessingPhase(
1, "result-stream", coreProjection, stateCache,
fakeCheckpointManager, fakeCheckpointManager);
}
public void setup0()
{
_bus = new InMemoryBus("bus");
_readDispatcher =
new RequestResponseDispatcher
<ClientMessage.ReadEventsBackwards, ClientMessage.ReadEventsBackwardsCompleted>(
_bus, e => e.CorrelationId, e => e.CorrelationId);
_writeDispatcher =
new RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted>(
_bus, e => e.CorrelationId, e => e.CorrelationId);
}
public override void TestFixtureSetUp()
{
base.TestFixtureSetUp();
_db = new TFChunkDb(new TFChunkDbConfig(PathName,
new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
16 * 1024,
0,
new InMemoryCheckpoint(),
new InMemoryCheckpoint(),
new InMemoryCheckpoint(-1),
new InMemoryCheckpoint(-1)));
_db.Open();
var chunk = _db.Manager.GetChunkFor(0);
_p1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res1 = chunk.TryAppend(_p1);
_c1 = LogRecord.Commit(_res1.NewPosition, Guid.NewGuid(), _p1.LogPosition, 0);
_cres1 = chunk.TryAppend(_c1);
_p2 = LogRecord.SingleWrite(_cres1.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res2 = chunk.TryAppend(_p2);
_c2 = LogRecord.Commit(_res2.NewPosition, Guid.NewGuid(), _p2.LogPosition, 1);
_cres2 = chunk.TryAppend(_c2);
_p3 = LogRecord.SingleWrite(_cres2.NewPosition,
Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
_res3 = chunk.TryAppend(_p3);
_c3 = LogRecord.Commit(_res3.NewPosition, Guid.NewGuid(), _p3.LogPosition, 2);
_cres3 = chunk.TryAppend(_c3);
chunk.Complete();
_db.Config.WriterCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
_db.Config.WriterCheckpoint.Flush();
_db.Config.ChaserCheckpoint.Write(chunk.ChunkHeader.ChunkEndPosition);
_db.Config.ChaserCheckpoint.Flush();
var bus = new InMemoryBus("Bus");
var ioDispatcher = new IODispatcher(bus, new PublishEnvelope(bus));
var scavenger = new TFChunkScavenger(_db, ioDispatcher, new FakeTableIndex(),
new FakeReadIndex(x => x == "es-to-scavenge"), Guid.NewGuid(), "fakeNodeIp");
scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: false);
_scavengedChunk = _db.Manager.GetChunk(0);
}
public void SetUp()
{
_stateCache = GivenStateCache();
_publisher = new InMemoryBus("test");
_coreProjection = new FakeCoreProjection();
_checkpointManager = new specification_with_multi_phase_core_projection.FakeCheckpointManager(
_publisher, Guid.NewGuid());
_resultStreamName = "result-stream";
_phase = new WriteQueryResultProjectionProcessingPhase(
1, _resultStreamName, _coreProjection, _stateCache, _checkpointManager, _checkpointManager);
When();
}
public ProjectionsStandardComponents(
int projectionWorkerThreadCount,
ProjectionType runProjections,
InMemoryBus masterOutputBus,
QueuedHandler masterInputQueue,
InMemoryBus masterMainBus)
{
_projectionWorkerThreadCount = projectionWorkerThreadCount;
_runProjections = runProjections;
_masterOutputBus = masterOutputBus;
_masterInputQueue = masterInputQueue;
_masterMainBus = masterMainBus;
}
public ProjectionWorkerNode(
Guid workerId,
TFChunkDb db,
QueuedHandler inputQueue,
ITimeProvider timeProvider,
ISingletonTimeoutScheduler timeoutScheduler,
ProjectionType runProjections)
{
_runProjections = runProjections;
Ensure.NotNull(db, "db");
_coreOutput = new InMemoryBus("Core Output");
IPublisher publisher = CoreOutput;
_subscriptionDispatcher = new ReaderSubscriptionDispatcher(publisher);
_spoolProcessingResponseDispatcher = new SpooledStreamReadingDispatcher(publisher);
_ioDispatcher = new IODispatcher(publisher, new PublishEnvelope(inputQueue));
_eventReaderCoreService = new EventReaderCoreService(
publisher,
_ioDispatcher,
10,
db.Config.WriterCheckpoint,
runHeadingReader: runProjections >= ProjectionType.System);
_feedReaderService = new FeedReaderService(_subscriptionDispatcher, timeProvider);
if (runProjections >= ProjectionType.System)
{
_projectionCoreServiceCommandReader = new ProjectionCoreServiceCommandReader(
publisher,
_ioDispatcher,
workerId.ToString("N"));
var multiStreamWriter = new MultiStreamMessageWriter(_ioDispatcher);
_slaveProjectionResponseWriter = new SlaveProjectionResponseWriter(multiStreamWriter);
_projectionCoreService = new ProjectionCoreService(
workerId,
inputQueue,
publisher,
_subscriptionDispatcher,
timeProvider,
_ioDispatcher,
_spoolProcessingResponseDispatcher,
timeoutScheduler);
var responseWriter = new ResponseWriter(_ioDispatcher);
_coreResponseWriter = new ProjectionCoreResponseWriter(responseWriter);
}
}
public static NodeEntryPoint StartWithOptions(NodeOptions options, Action<int> termination)
{
var slim = new ManualResetEventSlim(false);
var list = String.Join(Environment.NewLine,
options.GetPairs().Select(p => String.Format("{0} : {1}", p.Key, p.Value)));
Log.Info(list);
var bus = new InMemoryBus("OutputBus");
var controller = new NodeController(bus);
var mainQueue = new QueuedHandler(controller, "Main Queue");
controller.SetMainQueue(mainQueue);
Application.Start(i =>
{
slim.Set();
termination(i);
});
var http = new PlatformServerApiService(mainQueue, String.Format("http://{0}:{1}/", options.LocalHttpIp, options.HttpPort));
bus.Subscribe<SystemMessage.Init>(http);
bus.Subscribe<SystemMessage.StartShutdown>(http);
var timer = new TimerService(new ThreadBasedScheduler(new RealTimeProvider()));
bus.Subscribe<TimerMessage.Schedule>(timer);
// switch, based on configuration
AzureStoreConfiguration azureConfig;
if (AzureStoreConfiguration.TryParse(options.StoreLocation, out azureConfig))
{
var storageService = new AzureStorageService(azureConfig, mainQueue);
bus.Subscribe<ClientMessage.AppendEvents>(storageService);
bus.Subscribe<SystemMessage.Init>(storageService);
bus.Subscribe<ClientMessage.ImportEvents>(storageService);
bus.Subscribe<ClientMessage.RequestStoreReset>(storageService);
}
else
{
var storageService = new FileStorageService(options.StoreLocation, mainQueue);
bus.Subscribe<ClientMessage.AppendEvents>(storageService);
bus.Subscribe<SystemMessage.Init>(storageService);
bus.Subscribe<ClientMessage.ImportEvents>(storageService);
bus.Subscribe<ClientMessage.RequestStoreReset>(storageService);
}
mainQueue.Start();
mainQueue.Enqueue(new SystemMessage.Init());
return new NodeEntryPoint(mainQueue,slim);
}
public void setup0()
{
_bus = new InMemoryBus("bus");
_readDispatcher =
new RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted>(
_bus, e => e.CorrelationId, e => e.CorrelationId, new PublishEnvelope(_bus));
_writeDispatcher =
new RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted>(
_bus, e => e.CorrelationId, e => e.CorrelationId, new PublishEnvelope(_bus));
_bus.Subscribe(_readDispatcher);
_bus.Subscribe(_writeDispatcher);
_consumer = new TestHandler<Message>();
_bus.Subscribe(_consumer);
}
public async Task Publish_ToUnsubscribedEvent_NothingHappens()
{
var ev = new NewCommitteeEvent(Guid.NewGuid(), "Test Committee", "mandate");
bool hasRun = false;
Action<object, IDictionary<string, string>> action = (o, h) => hasRun = true;
using (var bus = new InMemoryBus())
{
await bus.Subscribe<NewDepartmentEvent>(action);
await bus.Publish(ev);
await bus.Start();
Thread.Sleep(20);
}
Assert.False(hasRun);
}
public async Task Publish_ToSubscribedEvent_RunsTask()
{
var ev = new NewCommitteeEvent(Guid.NewGuid(), "Test Committee", "mandate");
bool hasRun = false;
Action<object, IDictionary<string, string>> action = (o, h) => hasRun = true;
using (var bus = new InMemoryBus())
{
await bus.Subscribe<NewCommitteeEvent>(action);
await bus.Publish(ev);
await bus.Start();
Thread.Sleep(20);//give task time to complete
}
Assert.True(hasRun);
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestMessageHandler<ClientMessage.ReadStreamEventsBackward>();
_bus.Subscribe(_listEventsHandler);
_readDispatcher = new RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_writeDispatcher = new RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_bus.Subscribe(_readDispatcher);
_bus.Subscribe(_writeDispatcher);
_coreProjection = new CoreProjection(
"projection", Guid.NewGuid(), _bus, new FakeProjectionStateHandler(),
new ProjectionConfig(ProjectionMode.AdHoc, 5, 10, 1000, 250, true, true, true), _readDispatcher, _writeDispatcher);
_coreProjection.Start();
}
public void setup()
{
_bus = new InMemoryBus("bus");
_listEventsHandler = new TestHandler<ClientMessage.ReadStreamEventsBackward>();
_bus.Subscribe(_listEventsHandler);
_readDispatcher = new RequestResponseDispatcher
<ClientMessage.ReadStreamEventsBackward, ClientMessage.ReadStreamEventsBackwardCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_writeDispatcher = new RequestResponseDispatcher<ClientMessage.WriteEvents, ClientMessage.WriteEventsCompleted>(
_bus, v => v.CorrelationId, v => v.CorrelationId, new PublishEnvelope(_bus));
_bus.Subscribe(_readDispatcher);
_bus.Subscribe(_writeDispatcher);
IProjectionStateHandler projectionStateHandler = new FakeProjectionStateHandler();
_projectionConfig = new ProjectionConfig(5, 10, 1000, 250, true, true, false, false);
_coreProjection = CoreProjection.CreateAndPrepapre(
"projection", Guid.NewGuid(), _bus, projectionStateHandler, _projectionConfig, _readDispatcher,
_writeDispatcher, null);
_coreProjection.Start();
}
public void setup()
{
_bus = new InMemoryBus("bus");
_consumer = new WatchingConsumer();
_bus.Subscribe(_consumer);
_manager = new ProjectionManager(_bus, checkpointForStatistics: null);
_bus.Subscribe<ProjectionMessage.Projections.Stopped>(_manager);
_projectionName = "test-projection";
_manager.Handle(
new ProjectionManagementMessage.Post(
new PublishEnvelope(_bus), ProjectionMode.AdHoc, _projectionName, "JS",
@"fromAll(); on_any(function(){});log(1);", enabled: true));
// when
_newProjectionSource = @"fromAll(); on_any(function(){});log(2);";
_manager.Handle(
new ProjectionManagementMessage.UpdateQuery(
new PublishEnvelope(_bus), _projectionName, "JS", _newProjectionSource));
}
public void SetUp()
{
_bus = new InMemoryBus(string.Format("bus_{0}", _serverEndPoint.Port));
{
var pipelineBus = InMemoryBus.CreateTest();
var queue = new QueuedHandlerThreadPool(pipelineBus, "Test", true, TimeSpan.FromMilliseconds(50));
_multiQueuedHandler = new MultiQueuedHandler(new IQueuedHandler[]{queue}, null);
var httpAuthenticationProviders = new HttpAuthenticationProvider[] {new AnonymousHttpAuthenticationProvider()};
_service = new HttpService(ServiceAccessibility.Private, _bus, new NaiveUriRouter(),
_multiQueuedHandler, _serverEndPoint.ToHttpUrl());
HttpService.CreateAndSubscribePipeline(pipelineBus, httpAuthenticationProviders);
_client = new HttpAsyncClient();
}
HttpBootstrap.Subscribe(_bus, _service);
}
