public async Task smoke_test_able_to_fetch_a_page_of_events()
{
var list = new List <MembersJoined>();
for (int i = 0; i < 500; i++)
{
list.Add(new MembersJoined {
Day = i, Location = Guid.NewGuid().ToString(), Members = new string[] { Guid.NewGuid().ToString() }
});
}
using (var session = theStore.LightweightSession())
{
session.Events.Append(Guid.NewGuid(), list.ToArray());
await session.SaveChangesAsync().ConfigureAwait(false);
}
var options = new AsyncOptions();
var settings = new DaemonSettings
{
LeadingEdgeBuffer = 0.Seconds()
};
using (var data = new Fetcher(theStore, settings, options, Substitute.For <IDaemonLogger>(), new StubErrorHandler(), new Type[] { typeof(MembersJoined) }))
{
var page = await data.FetchNextPage(0).ConfigureAwait(false);
page.From.ShouldBe(0);
page.To.ShouldBe(options.PageSize);
page.Streams.SelectMany(x => x.Events).Count().ShouldBe(100);
}
}
public async Task run_with_error_handling()
{
StoreOptions(_ =>
{
_.Events.AsyncProjections.AggregateStreamsWith <ActiveProject>();
_.Events.AsyncProjections.Add(new OccasionalErroringProjection());
});
var settings = new DaemonSettings
{
LeadingEdgeBuffer = 0.Seconds()
};
settings.ExceptionHandling.OnException <DivideByZeroException>().Retry(3);
using (var daemon = theStore.BuildProjectionDaemon(logger: _logger, settings: settings))
{
daemon.StartAll();
await _fixture.PublishAllProjectEventsAsync(theStore);
//_fixture.PublishAllProjectEvents(theStore);
// Runs all projections until there are no more events coming in
await daemon.WaitForNonStaleResults().ConfigureAwait(false);
await daemon.StopAll().ConfigureAwait(false);
}
_fixture.CompareActiveProjects(theStore);
}
public void and_on_inner_exception_by_type_and_condition_2()
{
var settings = new DaemonSettings();
settings.OnException <InvalidOperationException>()
.AndInner(x => (x is CustomException {
ErrorCode: 500
}));
public void determine_continuation_for_skip_falls_back_to_stop_if_not_apply_event_exception()
{
var settings = new DaemonSettings();
settings.OnException <BadImageFormatException>().SkipEvent();
settings.DetermineContinuation(new BadImageFormatException(), 0)
.ShouldBeOfType <StopShard>();
}
public async Task correctly_correlates_by_stream()
{
var streams = new List <EventStream>();
var logger = new RecordingLogger();
using (var session = theStore.LightweightSession())
{
session.Logger = logger;
for (int i = 0; i < 20; i++)
{
var joined = new MembersJoined
{
Day = i,
Location = Guid.NewGuid().ToString(),
Members = new string[] { Guid.NewGuid().ToString() }
};
var departed = new MembersDeparted {
Day = i, Location = Guid.NewGuid().ToString(), Members = new[] { Guid.NewGuid().ToString() }
};
var reached = new ArrivedAtLocation {
Day = i, Location = Guid.NewGuid().ToString()
};
session.Events.Append(Guid.NewGuid(), joined, departed, reached);
}
await session.SaveChangesAsync().ConfigureAwait(false);
streams.AddRange(logger.LastCommit.GetStreams());
}
var types = new Type[]
{
typeof(MembersJoined), typeof(MembersDeparted), typeof(ArrivedAtLocation)
};
var settings = new DaemonSettings
{
LeadingEdgeBuffer = 0.Seconds()
};
using (var data = new Fetcher(theStore, settings, new AsyncOptions(), Substitute.For <IDaemonLogger>(), new StubErrorHandler(), types))
{
var page = await data.FetchNextPage(0).ConfigureAwait(false);
foreach (var stream in page.Streams)
{
var existing = streams.Single(x => x.Id == stream.Id);
existing.Events.Select(x => x.Id)
.ShouldHaveTheSameElementsAs(stream.Events.Select(x => x.Id));
}
}
}
public IDaemon CreateDaemon(ILogger logger, Type[] viewTypes = null, DaemonSettings settings = null, IProjection[] projections = null)
{
return(_store.BuildProjectionDaemon(
logger: new DaemonLogger(logger),
viewTypes: viewTypes,
settings: settings,
projections: projections
));
}
public void unknown_exception_type_just_gets_a_stop()
{
var settings = new DaemonSettings();
settings.OnException <BadImageFormatException>()
.RetryLater(1.Seconds(), 2.Seconds(), 3.Seconds());
settings.DetermineContinuation(new DivideByZeroException(), 0)
.ShouldBeOfType <StopShard>();
}
public void filter_by_exception_type_2()
{
var settings = new DaemonSettings();
settings.OnExceptionOfType(typeof(NpgsqlException));
var policy = settings.Policies.Single();
policy.Matches(new NpgsqlException()).ShouldBeTrue();
policy.Matches(new DivideByZeroException()).ShouldBeFalse();
}
public void filter_by_exception_type_and_more()
{
var settings = new DaemonSettings();
settings.OnException <CustomException>(x => x.ErrorCode == 200);
var policy = settings.Policies.Single();
policy.Matches(new NpgsqlException()).ShouldBeFalse();
policy.Matches(new CustomException(200)).ShouldBeTrue();
policy.Matches(new CustomException(201)).ShouldBeFalse();
}
public void and_on_inner_exception_by_type()
{
var settings = new DaemonSettings();
settings.OnException <InvalidOperationException>()
.AndInner <CustomException>();
var policy = settings.Policies.Single();
policy.Matches(new InvalidCastException()).ShouldBeFalse();
policy.Matches(new InvalidOperationException()).ShouldBeFalse();
policy.Matches(new InvalidOperationException("boom.", new DivideByZeroException())).ShouldBeFalse();
policy.Matches(new InvalidOperationException("boom.", new CustomException(111))).ShouldBeTrue();
}
// ReSharper disable once ContextualLoggerProblem
public HighWaterAgent(IHighWaterDetector detector, ShardStateTracker tracker, ILogger logger, DaemonSettings settings, CancellationToken token)
{
_detector = detector;
_tracker = tracker;
_logger = logger;
_settings = settings;
_token = token;
_timer = new Timer(_settings.HealthCheckPollingTime.TotalMilliseconds)
{
AutoReset = true
};
_timer.Elapsed += TimerOnElapsed;
}
public void determine_continuation_on_skip_with_apply_event_exception()
{
var settings = new DaemonSettings();
settings.OnApplyEventException().SkipEvent();
var @event = new Event <AEvent>(new AEvent())
{
Sequence = 55
};
settings.DetermineContinuation(new ApplyEventException(@event, new ArithmeticException()), 0)
.ShouldBeOfType <SkipEvent>()
.Event.Sequence.ShouldBe(@event.Sequence);
}
public void default_rules_for_npgsql_exception()
{
var settings = new DaemonSettings();
settings.DetermineContinuation(new NpgsqlException(), 0)
.ShouldBeOfType <RetryLater>();
settings.DetermineContinuation(new NpgsqlException(), 1)
.ShouldBeOfType <RetryLater>();
settings.DetermineContinuation(new NpgsqlException(), 2)
.ShouldBeOfType <RetryLater>();
settings.DetermineContinuation(new NpgsqlException(), 3)
.ShouldBeOfType <PauseShard>();
}
public Task StartAsync(CancellationToken cancellationToken)
{
Logger.Information("Projections starting");
var settings = new DaemonSettings
{
FetchingCooldown = 500.Milliseconds(),
LeadingEdgeBuffer = 100.Milliseconds()
};
_daemon = _store.BuildProjectionDaemon(
logger: new SerilogDaemonLogger(),
settings: settings
);
_daemon.StartAll();
return(Task.CompletedTask);
}
private static IDocumentStore SetupMarten()
{
IDocumentStore store = DocumentStore.For(_ =>
{
_.Connection(
"host=10.0.75.2;port=5433;database=daemon;password=postgres;username=postgres;MaxPoolSize=200;ConnectionIdleLifetime=30");
_.PLV8Enabled = false;
_.UseDefaultSerialization(EnumStorage.AsInteger, Casing.CamelCase);
_.DatabaseSchemaName = "public";
_.AutoCreateSchemaObjects = AutoCreate.CreateOrUpdate;
_.Events.AddEventTypes(new[]
{
typeof(EventType1), typeof(EventType2), typeof(EventType3), typeof(EventType4), typeof(EventType5), typeof(EventType6),
typeof(EventType7), typeof(EventType8), typeof(EventType9), typeof(EventType10), typeof(EventType11), typeof(EventType12)
});
_.Events.AsyncProjections.Add(new P1());
_.Events.AsyncProjections.Add(new P2());
_.Events.AsyncProjections.Add(new P3());
_.Events.AsyncProjections.Add(new P4());
_.Events.AsyncProjections.Add(new P5());
_.Events.AsyncProjections.Add(new P6());
_.Events.AsyncProjections.Add(new P7());
_.Events.AsyncProjections.Add(new P8());
_.Events.AsyncProjections.Add(new P9());
_.Events.AsyncProjections.Add(new P10());
_.Events.AsyncProjections.Add(new P11());
_.Events.AsyncProjections.Add(new P12());
});
store.Schema.ApplyAllConfiguredChangesToDatabase();
store.Schema.AssertDatabaseMatchesConfiguration();
var daemonSetting = new DaemonSettings {
LeadingEdgeBuffer = 0.Seconds()
};
daemonSetting.ExceptionHandling.OnException(e => true).Retry(3, 3.Seconds());
daemonSetting.ExceptionHandling.Cooldown = 10.Seconds();
var daemon = store.BuildProjectionDaemon(settings: daemonSetting);
daemon.StartAll();
return(store);
}
public void retry_logic_with_no_additional_continuation()
{
var settings = new DaemonSettings();
settings.OnException <BadImageFormatException>()
.RetryLater(1.Seconds(), 2.Seconds(), 3.Seconds());
settings.DetermineContinuation(new BadImageFormatException(), 0)
.ShouldBe(new RetryLater(1.Seconds()));
settings.DetermineContinuation(new BadImageFormatException(), 1)
.ShouldBe(new RetryLater(2.Seconds()));
settings.DetermineContinuation(new BadImageFormatException(), 2)
.ShouldBe(new RetryLater(3.Seconds()));
settings.DetermineContinuation(new BadImageFormatException(), 3)
.ShouldBeOfType <StopShard>();
}
public async Task filters_by_event_type_name()
{
using (var session = theStore.OpenSession())
{
for (int i = 0; i < 20; i++)
{
var joined = new MembersJoined
{
Day = i,
Location = Guid.NewGuid().ToString(),
Members = new string[] { Guid.NewGuid().ToString() }
};
var departed = new MembersDeparted {
Day = i, Location = Guid.NewGuid().ToString(), Members = new[] { Guid.NewGuid().ToString() }
};
var reached = new ArrivedAtLocation {
Day = i, Location = Guid.NewGuid().ToString()
};
session.Events.Append(Guid.NewGuid(), joined, departed, reached);
}
await session.SaveChangesAsync().ConfigureAwait(false);
}
var settings = new DaemonSettings
{
LeadingEdgeBuffer = 0.Seconds()
};
using (var data = new Fetcher(theStore, settings, new AsyncOptions(), Substitute.For <IDaemonLogger>(), new StubErrorHandler(), new Type[] { typeof(MembersDeparted), typeof(ArrivedAtLocation) }))
{
var page = await data.FetchNextPage(0).ConfigureAwait(false);
var all = page.Streams.SelectMany(x => x.Events).ToArray();
all.OfType <Event <MembersJoined> >().Any().ShouldBeFalse();
all.OfType <Event <MembersDeparted> >().Any().ShouldBeTrue();
all.OfType <Event <ArrivedAtLocation> >().Any().ShouldBeTrue();
}
}
public void retry_logic_then_pause()
{
var settings = new DaemonSettings();
settings.OnException <BadImageFormatException>()
.RetryLater(1.Seconds(), 2.Seconds(), 3.Seconds())
.Then.Pause(5.Seconds());
settings.DetermineContinuation(new BadImageFormatException(), 0)
.ShouldBe(new RetryLater(1.Seconds()));
settings.DetermineContinuation(new BadImageFormatException(), 1)
.ShouldBe(new RetryLater(2.Seconds()));
settings.DetermineContinuation(new BadImageFormatException(), 2)
.ShouldBe(new RetryLater(3.Seconds()));
settings.DetermineContinuation(new BadImageFormatException(), 3)
.ShouldBe(new PauseShard(5.Seconds()));
}
public async Task able_to_page_events()
{
var list = new List <MembersJoined>();
for (int i = 0; i < 500; i++)
{
list.Add(new MembersJoined {
Day = i, Location = Guid.NewGuid().ToString(), Members = new string[] { Guid.NewGuid().ToString() }
});
}
using (var session = theStore.LightweightSession())
{
session.Events.Append(Guid.NewGuid(), list.ToArray());
await session.SaveChangesAsync().ConfigureAwait(false);
}
var types = new Type[] { typeof(MembersJoined) };
var settings = new DaemonSettings
{
LeadingEdgeBuffer = 0.Seconds()
};
using (var data = new Fetcher(theStore, settings, new AsyncOptions(), Substitute.For <IDaemonLogger>(), new StubErrorHandler(), types))
{
var events1 = (await data.FetchNextPage(0).ConfigureAwait(false)).Streams.SelectMany(x => x.Events).ToArray();
var events2 = (await data.FetchNextPage(100).ConfigureAwait(false)).Streams.SelectMany(x => x.Events).ToArray();
var events3 = (await data.FetchNextPage(200).ConfigureAwait(false)).Streams.SelectMany(x => x.Events).ToArray();
events1.Intersect(events2).Any().ShouldBeFalse();
events1.Intersect(events3).Any().ShouldBeFalse();
events3.Intersect(events2).Any().ShouldBeFalse();
events1.Length.ShouldBe(100);
events2.Length.ShouldBe(100);
events3.Length.ShouldBe(100);
}
}
public override void SetUp()
{
_options = new StoreOptions();
_options.Connection(ConnectionSource.ConnectionString);
_options.Events.AsyncProjections.AggregateStreamsWith <ActiveProject>();
_options.Events.AsyncProjections.TransformEvents(new CommitViewTransform());
_settings = new DaemonSettings();
helper = Context.State.Retrieve <AsyncDaemonTestHelper>();
_store = new Lazy <DocumentStore>(() =>
{
var store = new DocumentStore(_options);
store.Advanced.Clean.CompletelyRemoveAll();
return(store);
});
_daemon = new Lazy <IDaemon>(() => _store.Value.BuildProjectionDaemon(settings: _settings));
}
/// <summary>
/// Convert Settings to DaemonSettings.
/// </summary>
/// <returns>DaemonSettings.</returns>
public DaemonSettings ToDaemonSettings()
{
DaemonSettings ret = new DaemonSettings();
ret.Database = Database;
ret.Logging = new DaemonSettings.LoggingSettings();
ret.Logging.ConsoleLogging = Logging.ConsoleLogging;
ret.Logging.FileDirectory = Logging.FileDirectory;
ret.Logging.FileLogging = Logging.FileLogging;
ret.Logging.Filename = Logging.Filename;
ret.Logging.Header = Logging.Header;
ret.Logging.MinimumLevel = Logging.MinimumLevel;
ret.Logging.SyslogServerIp = Logging.SyslogServerIp;
ret.Logging.SyslogServerPort = Logging.SyslogServerPort;
ret.SourceDocuments = SourceDocuments;
ret.ParsedDocuments = ParsedDocuments;
ret.TempStorage = TempStorage;
return(ret);
}
public async Task projectview_withdeleteevent_should_be_respected_during_projection_rebuild()
{
StoreOptions(_ =>
{
_.AutoCreateSchemaObjects = AutoCreate.All;
_.Events.InlineProjections.AggregateStreamsWith <Project>();
_.Events.ProjectView <Project, Guid>().DeleteEvent <ProjectClosed>();
});
var projectId = Guid.NewGuid();
theSession.Events.Append(projectId, new ProjectStarted {
Id = projectId, Name = "Marten"
});
await theSession.SaveChangesAsync();
theSession.Query <Project>().Count().ShouldBe(1);
theSession.Events.Append(projectId, new ProjectClosed {
Id = projectId
});
await theSession.SaveChangesAsync();
theSession.Query <Project>().Count().ShouldBe(0);
var settings = new DaemonSettings
{
LeadingEdgeBuffer = 0.Seconds()
};
using (var daemon = theStore.BuildProjectionDaemon(new[] { typeof(Project) }, settings: settings))
{
await daemon.RebuildAll();
}
//fails as user is back in the database after rebuilding
theSession.Query <Project>().Count().ShouldBe(0);
}
IDaemon IDocumentStore.BuildProjectionDaemon(Type[] viewTypes, IDaemonLogger logger, DaemonSettings settings,
IProjection[] projections)
{
return(_documentStore.BuildProjectionDaemon(viewTypes, logger, settings, projections));
}
private static void InitializeGlobals()
{
ConsoleColor prior = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.DarkGray;
Console.Write("| Initializing logging : ");
_Logging = new LoggingModule(
_Settings.Logging.SyslogServerIp,
_Settings.Logging.SyslogServerPort,
_Settings.Logging.ConsoleLogging);
if (_Settings.Logging.FileLogging && !String.IsNullOrEmpty(_Settings.Logging.Filename))
{
if (String.IsNullOrEmpty(_Settings.Logging.FileDirectory))
{
_Settings.Logging.FileDirectory = "./";
}
if (_Settings.Logging.FileDirectory.Contains("\\"))
{
_Settings.Logging.FileDirectory = _Settings.Logging.FileDirectory.Replace("\\", "/");
}
if (!Directory.Exists(_Settings.Logging.FileDirectory))
{
Directory.CreateDirectory(_Settings.Logging.FileDirectory);
}
_Logging.Settings.FileLogging = FileLoggingMode.FileWithDate;
_Logging.Settings.LogFilename = _Settings.Logging.FileDirectory + _Settings.Logging.Filename;
}
Console.WriteLine("[success]");
Console.Write("| Initializing Komodo daemon : ");
_DaemonSettings = _Settings.ToDaemonSettings();
_Daemon = new KomodoDaemon(_DaemonSettings);
Console.WriteLine("[success]");
Console.Write("| Initializing database : ");
_ORM = new WatsonORM(_Settings.Database.ToDatabaseSettings());
_ORM.InitializeDatabase();
_ORM.InitializeTable(typeof(ApiKey));
_ORM.InitializeTable(typeof(Index));
_ORM.InitializeTable(typeof(Metadata));
_ORM.InitializeTable(typeof(MetadataDocument));
_ORM.InitializeTable(typeof(Node));
_ORM.InitializeTable(typeof(ParsedDocument));
_ORM.InitializeTable(typeof(Permission));
_ORM.InitializeTable(typeof(SourceDocument));
_ORM.InitializeTable(typeof(TermDoc));
_ORM.InitializeTable(typeof(TermGuid));
_ORM.InitializeTable(typeof(User));
Console.WriteLine("[success]");
Console.Write("| Initializing authentication : ");
_Auth = new AuthManager(_ORM);
Console.WriteLine("[success]");
Console.Write("| Initializing webserver : ");
_Webserver = new WatsonWebserver.Server(
_Settings.Server.ListenerHostname,
_Settings.Server.ListenerPort,
_Settings.Server.Ssl,
RequestReceived);
_Webserver.Routes.Content.Add("/Assets/", true);
_Webserver.Settings.AccessControl.Mode = AccessControlMode.DefaultPermit;
_Webserver.Start();
Console.WriteLine(
"[success] " +
(_Settings.Server.Ssl ? "https://" : "http://") +
_Settings.Server.ListenerHostname + ":" + _Settings.Server.ListenerPort);
if (_Settings.Server.ListenerHostname.Equals("localhost") || _Settings.Server.ListenerHostname.Equals("127.0.0.1"))
{
// 1 2 3 4 5 6 7 8
// 12345678901234567890123456789012345678901234567890123456789012345678901234567890
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("");
Console.WriteLine("WARNING: Komodo started on '" + _Settings.Server.ListenerHostname + "'");
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("Komodo can only service requests from the local machine. If you wish to serve");
Console.WriteLine("external requests, edit the system.json file and specify a DNS-resolvable");
Console.WriteLine("hostname in the Server.ListenerHostname field.");
Console.WriteLine("");
}
List <string> adminListeners = new List <string> {
"*", "+", "0.0.0.0"
};
if (adminListeners.Contains(_Settings.Server.ListenerHostname))
{
// 1 2 3 4 5 6 7 8
// 12345678901234567890123456789012345678901234567890123456789012345678901234567890
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("");
Console.WriteLine("NOTICE: Komodo is listening on a wildcard hostname: '" + _Settings.Server.ListenerHostname + "'");
Console.ForegroundColor = ConsoleColor.DarkCyan;
Console.WriteLine("Komodo must be run with administrative privileges, otherwise it will not be able");
Console.WriteLine("to respond to incoming requests.");
Console.WriteLine("");
}
Console.ForegroundColor = ConsoleColor.DarkGray;
if (_Settings.EnableConsole)
{
_Console = new ConsoleManager(_Settings, ExitApplication);
}
Console.ForegroundColor = prior;
Console.WriteLine("");
}
public IDaemon BuildProjectionDaemon(Type[] viewTypes = null, IDaemonLogger logger = null, DaemonSettings settings = null, IProjection[] projections = null)
{
Tenancy.Default.EnsureStorageExists(typeof(EventStream));
if (projections == null)
{
projections = viewTypes?.Select(x => Events.ProjectionFor(x)).Where(x => x != null).ToArray() ?? Events.AsyncProjections.ToArray();
}
return(new Daemon(this, Tenancy.Default, settings ?? new DaemonSettings(), logger ?? new NulloDaemonLogger(), projections));
}
public ScheduleHostedService(IOptions <DaemonSettings> settings, ILogger <ScheduleHostedService> logger)
{
_logger = logger;
_settings = settings.Value;
}
internal ExceptionPolicy(DaemonSettings parent, Func <Exception, bool> filter)
{
_parent = parent;
_filters.Add(filter);
}
