void CleanupDispatchedEnvelope(ImmutableEnvelope envelope)
{
try
{
_manager.Memorize(envelope.EnvelopeId);
}
catch (ThreadAbortException)
{
// continue;
throw;
}
catch (Exception ex)
{
SystemObserver.Notify(new EnvelopeCleanupFailed(ex, _dispatcherName, envelope));
}
try
{
_quarantine.TryRelease(envelope);
}
catch (ThreadAbortException)
{
// continue
throw;
}
catch (Exception ex)
{
SystemObserver.Notify(new EnvelopeCleanupFailed(ex, _dispatcherName, envelope));
}
SystemObserver.Notify(new EnvelopeDispatched(envelope, _dispatcherName));
}
static void ObserveWhileCan(IEnumerable <StoreRecord> records, RedirectToDynamicEvent wire, CancellationToken token)
{
var watch = Stopwatch.StartNew();
int count = 0;
foreach (var record in records)
{
count += 1;
if (token.IsCancellationRequested)
{
return;
}
if (count % 50000 == 0)
{
SystemObserver.Notify("Observing {0} {1}", count,
Math.Round(watch.Elapsed.TotalSeconds, 2));
watch.Restart();
}
foreach (var message in record.Items)
{
if (message is IEvent)
{
wire.InvokeEvent(message);
}
}
}
}
private long ObserveWhileCan(IEnumerable <StoreRecord> records, RedirectToDynamicEvent wire, CancellationToken token)
{
var watch = Stopwatch.StartNew();
var count = 0;
long eventStoreVersion = 0;
foreach (var record in records)
{
count += 1;
if (token.IsCancellationRequested)
{
return(eventStoreVersion);
}
if (count % 50000 == 0)
{
SystemObserver.Notify("[observing]\t{0}\t{1} records in {2} seconds", this._name, count, Math.Round(watch.Elapsed.TotalSeconds, 2));
watch.Restart();
}
foreach (var message in record.Items.OfType <TEvent>())
{
wire.InvokeEvent(message);
}
eventStoreVersion = record.StoreVersion;
}
return(eventStoreVersion);
}
/// <summary>
/// Builds this <see cref="CqrsEngineHost"/>.
/// </summary>
/// <returns>new instance of cloud engine host</returns>
public CqrsEngineHost Build()
{
// nonconditional registrations
// System presets
_builder.RegisterType <DispatcherProcess>();
_builder.RegisterInstance(new MemoryAccount());
foreach (var module in _moduleEnlistments)
{
_builder.RegisterModule(module);
}
var container = _builder.Build();
var system = new SystemObserver(_observers.ToArray());
var reg = container.ComponentRegistry;
Configure(reg, system);
var processes = container.Resolve <IEnumerable <IEngineProcess> >();
var scope = container.Resolve <ILifetimeScope>();
var host = new CqrsEngineHost(scope, system, processes);
host.Initialize();
return(host);
}
public Task Start(CancellationToken token)
{
var tasks = _serverProcesses.Select(p => p.Start(token)).ToArray();
if (tasks.Length == 0)
{
throw new InvalidOperationException(string.Format("There were no instances of '{0}' registered", typeof(IEngineProcess).Name));
}
var names =
_serverProcesses.Select(p => string.Format("{0}({1:X8})", p.GetType().Name, p.GetHashCode())).ToArray();
SystemObserver.Notify(new EngineStarted(names));
return(Task.Factory.StartNew(() =>
{
var watch = Stopwatch.StartNew();
try
{
Task.WaitAll(tasks, token);
}
catch (OperationCanceledException)
{}
SystemObserver.Notify(new EngineStopped(watch.Elapsed));
}));
}
private static async Task WaitForViewsToSave(string bucket, List <Task> tasks, int counter, Stopwatch sw)
{
SystemObserver.Notify("{0}: Added {1}({2}) records to save", bucket, tasks.Count, counter);
await Task.WhenAll(tasks);
sw.Stop();
SystemObserver.Notify("{0}: Total {1}({2}) records saved in {3}", bucket, tasks.Count, counter, sw.Elapsed);
}
internal void Initialize(CancellationToken token)
{
foreach (var process in _serverProcesses)
{
process.Initialize(token);
}
SystemObserver.Notify(new EngineInitialized());
}
public CqrsEngineHost(
ILifetimeScope container,
SystemObserver observer,
IEnumerable <IEngineProcess> serverProcesses)
{
Container = container;
_serverProcesses = serverProcesses;
_observer = observer;
}
static void ConfigureObserver()
{
Trace.Listeners.Add(new ConsoleTraceListener());
var observer = new ConsoleObserver();
SystemObserver.Swap(observer);
Context.SwapForDebug(s => SystemObserver.Notify(s));
}
internal void Initialize()
{
SystemObserver.Notify(new EngineInitializationStarted());
foreach (var process in _serverProcesses)
{
process.Initialize();
}
SystemObserver.Notify(new EngineInitialized());
}
public bool TryToQuarantine(ImmutableEnvelope context, Exception ex)
{
// quit immediately, we don't want an endless cycle!
//if (context.Items.Any(m => m.Content is MessageQuarantined))
// return true;
var quarantined = _quarantine.TryToQuarantine(context, ex);
try
{
var file = string.Format("{0:yyyy-MM-dd}-{1}-engine.txt",
DateTime.UtcNow,
context.EnvelopeId.ToLowerInvariant());
var data = "";
if (_root.Exists(file))
{
using (var stream = _root.OpenRead(file))
using (var reader = new StreamReader(stream))
{
data = reader.ReadToEnd();
}
}
var builder = new StringBuilder(data);
if (builder.Length == 0)
{
DescribeMessage(builder, context);
}
builder.AppendLine("[Exception]");
builder.AppendLine(DateTime.UtcNow.ToString(CultureInfo.InvariantCulture));
builder.AppendLine(ex.ToString());
builder.AppendLine();
var text = builder.ToString();
using (var stream = _root.OpenWrite(file))
using (var writer = new StreamWriter(stream))
{
writer.Write(text);
}
if (quarantined)
{
ReportFailure(text, context);
}
}
catch (Exception x)
{
SystemObserver.Notify(x.ToString());
}
return(quarantined);
}
public void ShouldReceiveModifiedEntity()
{
Filter testFilter = new Filter().AnyOf(typeof(FirstTestComponent));
testSystem.filter = testFilter;
SystemObserver.Subscribe(testSystem);
testEntity.AddComponent(new FirstTestComponent());
Assert.AreSame(testEntity, testSystem.receivedEntity);
}
public void ShouldTriggerSystemIfComponentIsAddedAndMatches()
{
Filter testFilter = new Filter().AllOf(typeof(FirstTestComponent), typeof(SecondTestComponent));
testSystem.filter = testFilter;
SystemObserver.Subscribe(testSystem);
testEntity.AddComponent(new FirstTestComponent());
Assert.IsFalse(testSystem.isTriggered);
testEntity.AddComponent(new SecondTestComponent());
Assert.IsTrue(testSystem.isTriggered);
}
public void WriteContents(string bucket, IEnumerable <DocumentRecord> records)
{
var retryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(0.5), 100);
var directory = this._client.GetBlobDirectory(bucket);
directory.Container.CreateIfNotExists();
var tasks = new List <Task>();
var fullSw = new Stopwatch();
var fullCounter = 0;
try
{
var counter = 0;
var sw = new Stopwatch();
fullSw.Start();
sw.Start();
foreach (var record in records)
{
var data = record.Read();
var blob = directory.GetBlockBlobReference(record.Key);
tasks.Add(AP.LongAsync.Do(async() => await blob.UploadFromByteArrayAsync(data, 0, data.Length, null, new BlobRequestOptions {
RetryPolicy = retryPolicy, MaximumExecutionTime = TimeSpan.FromMinutes(15), ServerTimeout = TimeSpan.FromMinutes(15)
}, null).ConfigureAwait(false)));
counter++;
fullCounter++;
if (tasks.Count == 200)
{
SystemObserver.Notify("{0}: Added {1}({2}) records to save", bucket, tasks.Count, counter);
Task.WaitAll(tasks.ToArray());
sw.Stop();
SystemObserver.Notify("{0}: Total {1}({2}) records saved in {3}", bucket, tasks.Count, counter, sw.Elapsed);
tasks.Clear();
sw.Restart();
}
}
SystemObserver.Notify("{0}: Added {1}({2}) records to save", bucket, tasks.Count, counter);
Task.WaitAll(tasks.ToArray());
sw.Stop();
SystemObserver.Notify("{0}: Total {1}({2}) records saved in {3}", bucket, tasks.Count, counter, sw.Elapsed);
}
finally
{
fullSw.Stop();
SystemObserver.Notify("{0}: Saved total {1} records in {2}", bucket, fullCounter, fullSw.Elapsed);
}
}
static void CallHandlers(RedirectToCommand serviceCommands, ImmutableEnvelope aem)
{
var content = aem.Message;
var watch = Stopwatch.StartNew();
serviceCommands.Invoke(content);
watch.Stop();
var seconds = watch.Elapsed.TotalSeconds;
if (seconds > 10)
{
SystemObserver.Notify("[Warn]: {0} took {1:0.0} seconds", content.GetType().Name, seconds);
}
}
public Task Start(CancellationToken token)
{
var virtualDirectory = _environment.VirtualDirectory ?? "";
if (!virtualDirectory.EndsWith("/"))
{
virtualDirectory += "/";
}
var prefix = "http://" + (_environment.OptionalHostName ?? "+") + ":" + _environment.Port +
virtualDirectory;
return(Task.Factory.StartNew(() =>
{
try
{
_listener = new HttpListener();
_listener.Prefixes.Add(prefix);
_listener.Start();
}
catch (HttpListenerException ex)
{
if (ex.ErrorCode == 5)
{
var solution =
string.Format(
"Make sure to run as admin or reserve prefix. For reservation you can try executing as admin: netsh http add urlacl url={0} user={1}\\{2}",
prefix, Environment.MachineName,
Environment.UserName);
SystemObserver.Notify(new ConfigurationWarningEncountered("HttpListener can't connect to " + prefix + ". " + solution,
ex));
}
else
{
SystemObserver.Notify(new FailedToStartHttpListener(ex, prefix));
}
}
catch (Exception ex)
{
SystemObserver.Notify(new FailedToStartHttpListener(ex, prefix));
}
_listener.BeginGetContext(ar => GetContext(ar, token), null);
WaitTillAllProcessorsQuit(token);
_listener.Stop();
}, token));
}
public static Container BuildEnvironment()
{
//JsConfig.DateHandler = JsonDateHandler.ISO8601;
ConfigureObserver();
var integrationPath = AzureSettingsProvider.GetStringOrThrow(Conventions.StorageConfigName);
//var email = AzureSettingsProvider.GetStringOrThrow(Conventions.SmtpConfigName);
//var core = new SmtpHandlerCore(email);
var setup = new Setup
{
//Smtp = core,
//FreeApiKey = freeApiKey,
//WebClientUrl = clientUri,
//HttpEndpoint = endPoint,
//EncryptorTool = new EncryptorTool(systemKey)
};
if (integrationPath.StartsWith("file:"))
{
var path = integrationPath.Remove(0, 5);
SystemObserver.Notify("Using store : {0}", path);
var config = FileStorage.CreateConfig(path);
setup.Streaming = config.CreateStreaming();
setup.DocumentStoreFactory = config.CreateDocumentStore;
setup.QueueReaderFactory = s => config.CreateInbox(s, DecayEvil.BuildExponentialDecay(500));
setup.QueueWriterFactory = config.CreateQueueWriter;
setup.AppendOnlyStoreFactory = config.CreateAppendOnlyStore;
setup.ConfigureQueues(1, 1);
return(setup.Build());
}
if (integrationPath.StartsWith("Default") || integrationPath.Equals("UseDevelopmentStorage=true", StringComparison.InvariantCultureIgnoreCase))
{
var config = AzureStorage.CreateConfig(integrationPath);
setup.Streaming = config.CreateStreaming();
setup.DocumentStoreFactory = config.CreateDocumentStore;
setup.QueueReaderFactory = s => config.CreateQueueReader(s);
setup.QueueWriterFactory = config.CreateQueueWriter;
setup.AppendOnlyStoreFactory = config.CreateAppendOnlyStore;
setup.ConfigureQueues(4, 4);
return(setup.Build());
}
throw new InvalidOperationException("Unsupported environment");
}
void Configure(IComponentRegistry reg)
{
var system = new SystemObserver(_observers.ToArray());
reg.Register <ISystemObserver>(system);
// domain should go before serialization
_domain.Configure(reg, _serializationList);
_storageModule.Configure(reg);
var serializer = _dataSerializer(_serializationList.GetAndMakeReadOnly());
var streamer = new EnvelopeStreamer(_envelopeSerializer, serializer);
reg.Register(new MemoryAccount());
reg.Register(serializer);
reg.Register <IEnvelopeStreamer>(c => streamer);
reg.Register(_registry);
}
private void CallHandlers(IEnumerable <object> recordItems)
{
foreach (var e in recordItems.OfType <TEvent>())
{
var watch = Stopwatch.StartNew();
this._eventHandlers.InvokeEvent(e);
watch.Stop();
var seconds = watch.Elapsed.TotalSeconds;
if (seconds > 10)
{
SystemObserver.Notify("[Warn] {0}\t{1}\t{2} took {3:0.0} seconds for handlers to process event {4}",
this._name, this._projectionName, e.GetType().Name, seconds, e.ToString());
}
}
}
public void Dispatch(byte[] message)
{
ImmutableEnvelope envelope = null;
try
{
envelope = _streamer.ReadAsEnvelopeData(message);
}
catch (Exception ex)
{
// permanent quarantine for serialization problems
_quarantine.Quarantine(message, ex);
SystemObserver.Notify(new EnvelopeDeserializationFailed(ex, "dispatch"));
return;
}
if (_manager.DoWeRemember(envelope.EnvelopeId))
{
SystemObserver.Notify(new EnvelopeDuplicateDiscarded(envelope.EnvelopeId));
return;
}
try
{
_action(envelope);
// non-essential but recommended
CleanupDispatchedEnvelope(envelope);
}
catch (ThreadAbortException)
{
return;
}
catch (Exception ex)
{
if (_quarantine.TryToQuarantine(envelope, ex))
{
SystemObserver.Notify(new EnvelopeQuarantined(ex, _dispatcherName, envelope));
// message quarantined. Swallow
return;
}
// if we are on a persistent queue, this will tell to retry
throw;
}
}
public async Task WriteContentsAsync(string bucket, IEnumerable <DocumentRecord> records, CancellationToken token)
{
var retryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(0.5), 100);
var directory = this._client.GetBlobDirectory(bucket);
await directory.Container.CreateIfNotExistsAsync(token);
var tasks = new List <Task>();
var fullSw = new Stopwatch();
var fullCounter = 0;
try
{
var counter = 0;
var sw = new Stopwatch();
fullSw.Start();
sw.Start();
foreach (var record in records)
{
var data = record.Read();
var blob = directory.GetBlockBlobReference(record.Key);
tasks.Add(AP.LongAsync.Do(() => blob.UploadFromByteArrayAsync(data, 0, data.Length, null, new BlobRequestOptions {
RetryPolicy = retryPolicy, MaximumExecutionTime = TimeSpan.FromMinutes(15), ServerTimeout = TimeSpan.FromMinutes(15)
}, null, token)));
counter++;
fullCounter++;
if (tasks.Count < 200)
{
continue;
}
await WaitForViewsToSave(bucket, tasks, counter, sw);
tasks.Clear();
sw.Restart();
}
await WaitForViewsToSave(bucket, tasks, counter, sw);
}
finally
{
fullSw.Stop();
SystemObserver.Notify("{0}: Saved total {1} records in {2}", bucket, fullCounter, fullSw.Elapsed);
}
}
public async Task WriteContentsAsync(string bucket, IEnumerable <DocumentRecord> records, CancellationToken token)
{
var buck = Path.Combine(this._folderPath, bucket);
if (!Directory.Exists(buck))
{
Directory.CreateDirectory(buck);
}
var tasks = new List <Task>();
var fullSw = new Stopwatch();
var fullCounter = 0;
var counter = 0;
var sw = new Stopwatch();
fullSw.Start();
sw.Start();
foreach (var record in records)
{
var recordPath = Path.Combine(buck, record.Key);
var path = Path.GetDirectoryName(recordPath) ?? "";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
var data = record.Read();
tasks.Add(AP.LongAsync.Do(() => this.SaveData(recordPath, data)));
counter++;
fullCounter++;
if (tasks.Count == 1000)
{
await WaitForViewsSave(bucket, tasks, counter, sw);
tasks.Clear();
sw.Restart();
}
}
await WaitForViewsSave(bucket, tasks, counter, sw);
fullSw.Stop();
SystemObserver.Notify("{0}: Saved total {1} records in {2}", bucket, fullCounter, fullSw.Elapsed);
}
void ProcessMessage(MessageTransportContext context)
{
var dispatched = false;
try
{
_dispatcher(context.Unpacked);
dispatched = true;
}
catch (ThreadAbortException)
{
// we are shutting down. Stop immediately
return;
}
catch (Exception dispatchEx)
{
// if the code below fails, it will just cause everything to be reprocessed later,
// which is OK (duplication manager will handle this)
SystemObserver.Notify(new MessageDispatchFailed(context, context.QueueName, dispatchEx));
// quarantine is atomic with the processing
_inbox.TryNotifyNack(context);
}
if (!dispatched)
{
return;
}
try
{
_inbox.AckMessage(context);
// 3rd - notify.
SystemObserver.Notify(new MessageAcked(context));
}
catch (ThreadAbortException)
{
// nothing. We are going to sleep
}
catch (Exception ex)
{
// not a big deal. Message will be processed again.
SystemObserver.Notify(new MessageAckFailed(ex, context));
}
}
public GetEnvelopeResult TryGetMessage()
{
CloudQueueMessage message;
try
{
message = this._queue.GetMessage(this._visibilityTimeout);
}
catch (ThreadAbortException)
{
// we are stopping
return(GetEnvelopeResult.Empty);
}
catch (Exception ex)
{
SystemObserver.Notify(new FailedToReadMessage(ex, this._queueName));
return(GetEnvelopeResult.Error());
}
if (null == message)
{
return(GetEnvelopeResult.Empty);
}
try
{
var unpacked = this.DownloadPackage(message);
return(GetEnvelopeResult.Success(unpacked));
}
catch (StorageException ex)
{
SystemObserver.Notify(new FailedToAccessStorage(ex, this._queue.Name, message.Id));
return(GetEnvelopeResult.Retry);
}
catch (Exception ex)
{
SystemObserver.Notify(new MessageInboxFailed(ex, this._queue.Name, message.Id));
// new poison details
this._posionQueue.Value.AddMessage(message);
this._queue.DeleteMessage(message);
return(GetEnvelopeResult.Retry);
}
}
public GetEnvelopeResult TryGetMessage()
{
FileInfo message;
try
{
message = _queue.EnumerateFiles().FirstOrDefault();
}
catch (Exception ex)
{
SystemObserver.Notify(new FailedToReadMessage(ex, _queueName));
return(GetEnvelopeResult.Error());
}
if (null == message)
{
return(GetEnvelopeResult.Empty);
}
try
{
var buffer = File.ReadAllBytes(message.FullName);
var unpacked = new MessageTransportContext(message, buffer, _queueName);
return(GetEnvelopeResult.Success(unpacked));
}
catch (IOException ex)
{
// this is probably sharing violation, no need to
// scare people.
if (!IsSharingViolation(ex))
{
SystemObserver.Notify(new FailedToAccessStorage(ex, _queue.Name, message.Name));
}
return(GetEnvelopeResult.Retry);
}
catch (Exception ex)
{
SystemObserver.Notify(new MessageInboxFailed(ex, _queue.Name, message.FullName));
// new poison details
return(GetEnvelopeResult.Retry);
}
}
private async Task RebuildProjection(long eventStoreVersion, ProjectionInfo projectionInfo, IDocumentStore memoryContainer, CancellationToken cancellationToken)
{
await projectionInfo.Reset();
var saveTasks = new List <Task>(projectionInfo.ViewBuckets.Length);
foreach (var bucket in projectionInfo.ViewBuckets)
{
var sw = new Stopwatch();
sw.Start();
var contents = memoryContainer.EnumerateContents(bucket);
var bucket1 = bucket;
saveTasks.Add(this._targetContainer.WriteContentsAsync(bucket, contents, cancellationToken).ContinueWith(t =>
{
sw.Stop();
SystemObserver.Notify("[good]\t{0}\t{1}\t{2:g}\tSaved View bucket {3}", this._name, projectionInfo.ProjectionName, sw.Elapsed, bucket1);
}, cancellationToken, TaskContinuationOptions.OnlyOnRanToCompletion, TaskScheduler.Current));
}
await Task.WhenAll(saveTasks);
await projectionInfo.UpdateEventStreamVersion(eventStoreVersion);
}
private Task RebuildProjections(List <ProjectionInfo> needRebuild, IDocumentStore memoryContainer, CancellationToken token)
{
if (needRebuild.Count == 0)
{
return(Task.FromResult(true));
}
var updateTimer = new Stopwatch();
updateTimer.Start();
// observe projections
var watch = Stopwatch.StartNew();
var wire = new RedirectToDynamicEvent();
needRebuild.ForEach(x => wire.WireToWhen(x.GetTempProjection()));
var handlersWatch = Stopwatch.StartNew();
var eventStoreVersion = this.ObserveWhileCan(this._eventsStore.EnumerateAllItems(0, int.MaxValue), wire, token);
if (token.IsCancellationRequested)
{
SystemObserver.Notify("[warn]\t{0}\tShutdown. Aborting projections before anything was changed.", this._name);
return(Task.FromResult(true));
}
var timeTotal = watch.Elapsed.TotalSeconds;
var handlerTicks = handlersWatch.ElapsedTicks;
var timeInHandlers = Math.Round(TimeSpan.FromTicks(handlerTicks).TotalSeconds, 1);
SystemObserver.Notify("[observe]\t{2}\t{0}sec ({1}sec in handlers) - Replayed events from", Math.Round(timeTotal, 0), timeInHandlers, this._name);
var rebuildTasks = needRebuild.Select(projectionInfo => this.RebuildProjection(eventStoreVersion, projectionInfo, memoryContainer, token));
return(Task.WhenAll(rebuildTasks));
}
public void SendEnvelope(ImmutableEnvelope envelope)
{
var queue = GetOutboundQueue();
var data = _streamer.SaveEnvelopeData(envelope);
if (Transaction.Current == null)
{
queue.PutMessage(data);
SystemObserver.Notify(new EnvelopeSent(queue.Name, envelope.EnvelopeId, false,
envelope.Items.Select(x => x.MappedType.Name).ToArray(), envelope.GetAllAttributes()));
}
else
{
var action = new CommitActionEnlistment(() =>
{
queue.PutMessage(data);
SystemObserver.Notify(new EnvelopeSent(queue.Name, envelope.EnvelopeId, true,
envelope.Items.Select(x => x.MappedType.Name).ToArray(), envelope.GetAllAttributes()));
});
Transaction.Current.EnlistVolatile(action, EnlistmentOptions.None);
}
}
void ReceiveMessages(CancellationToken outer)
{
using (var source = CancellationTokenSource.CreateLinkedTokenSource(_disposal.Token, outer))
{
while (true)
{
MessageTransportContext context;
try
{
if (!_inbox.TakeMessage(source.Token, out context))
{
// we didn't retrieve queue within the token lifetime.
// it's time to shutdown the server
break;
}
}
catch (Exception)
{
continue;
}
try
{
ProcessMessage(context);
}
catch (ThreadAbortException)
{
// Nothing. we are being shutdown
}
catch (Exception ex)
{
var e = new DispatchRecoveryFailed(ex, context, context.QueueName);
SystemObserver.Notify(e);
}
}
}
}
static void Observe(ITapeStream tapes, RedirectToDynamicEvent wire)
{
var date = DateTime.MinValue;
var watch = Stopwatch.StartNew();
foreach (var record in tapes.ReadRecords(0, int.MaxValue))
{
var env = Streamer.ReadAsEnvelopeData(record.Data);
if (date.Month != env.CreatedOnUtc.Month)
{
date = env.CreatedOnUtc;
SystemObserver.Notify("Observing {0:yyyy-MM-dd} {1}", date, Math.Round(watch.Elapsed.TotalSeconds, 2));
watch.Restart();
}
foreach (var item in env.Items)
{
var e = item.Content as IEvent;
if (e != null)
{
wire.InvokeEvent(e);
}
}
}
}
