public Task <bool> BeginTransaction(string incomingFilePath)
{
Directory.CreateDirectory(transactionDir);
FileToProcess = Path.Combine(transactionDir, Path.GetFileName(incomingFilePath));
return(AsyncFile.Move(incomingFilePath, FileToProcess));
}
public Task <bool> BeginTransaction(string incomingFilePath, CancellationToken cancellationToken)
{
Directory.CreateDirectory(processingDirectory);
FileToProcess = Path.Combine(processingDirectory, Path.GetFileName(incomingFilePath));
return(AsyncFile.Move(incomingFilePath, FileToProcess, cancellationToken));
}
async Task <IEnumerable <string> > GetSubscribersFor(Type messageType)
{
var subscribers = new HashSet <string>();
var allEventTypes = GetPotentialEventTypes(messageType);
foreach (var eventType in allEventTypes)
{
var eventDir = Path.Combine(basePath, ".events", eventType.FullName);
if (!Directory.Exists(eventDir))
{
continue;
}
foreach (var file in Directory.GetFiles(eventDir))
{
var allText = await AsyncFile.ReadText(file)
.ConfigureAwait(false);
subscribers.Add(allText);
}
}
return(subscribers);
}
async Task Subscribe(MessageMetadata eventType, CancellationToken cancellationToken)
{
var eventDir = GetEventDirectory(eventType.MessageType);
// the subscription directory and the subscription information will be created no matter if there's a publisher for the event assuming that the publisher haven’t started yet
Directory.CreateDirectory(eventDir);
var subscriptionEntryPath = GetSubscriptionEntryPath(eventDir);
var attempts = 0;
// since we have a design that can run into concurrency exceptions we perform a few retries
while (true)
{
try
{
await AsyncFile.WriteText(subscriptionEntryPath, localAddress, cancellationToken).ConfigureAwait(false);
return;
}
catch (IOException)
{
attempts++;
if (attempts > 10)
{
throw;
}
//allow the other task to complete
await Task.Delay(100, cancellationToken).ConfigureAwait(false);
}
}
}
public async Task Subscribe(Type eventType, ContextBag context)
{
var eventDir = GetEventDirectory(eventType);
// that way we can detect that there is indeed a publisher for the event. That said it also means that we will have do "retries" here due to race condition.
Directory.CreateDirectory(eventDir);
var subscriptionEntryPath = GetSubscriptionEntryPath(eventDir);
var attempts = 0;
// since we have a design that can run into concurrency exceptions we perform a few retries
while (true)
{
try
{
await AsyncFile.WriteText(subscriptionEntryPath, localAddress).ConfigureAwait(false);
return;
}
catch (IOException)
{
attempts++;
if (attempts > 10)
{
throw;
}
//allow the other task to complete
await Task.Delay(100).ConfigureAwait(false);
}
}
}
static Func <string, Task> BuildDefaultDiagnosticsWriter(ReadOnlySettings settings)
{
if (!settings.TryGet <string>(DiagnosticSettingsExtensions.DiagnosticsPathKey, out var diagnosticsRootPath))
{
try
{
diagnosticsRootPath = Path.Combine(Host.GetOutputDirectory(), ".diagnostics");
}
catch (Exception e)
{
logger.Error("Unable to determine the diagnostics output directory. Check the attached exception for further information, or configure a custom diagnostics directory using 'EndpointConfiguration.SetDiagnosticsPath()'.", e);
}
}
if (!Directory.Exists(diagnosticsRootPath))
{
Directory.CreateDirectory(diagnosticsRootPath);
}
var endpointName = settings.EndpointName();
// Once we have the proper hosting model in place we can skip the endpoint name since the host would
// know how to handle multi hosting but for now we do this so that multi-hosting users will get a file per endpoint
var startupDiagnosticsFileName = $"{endpointName}-configuration.txt";
var startupDiagnosticsFilePath = Path.Combine(diagnosticsRootPath, startupDiagnosticsFileName);
return(data => AsyncFile.WriteText(startupDiagnosticsFilePath, data));
}
public Task Enlist(string messagePath, string messageContents)
{
var inProgressFileName = Path.GetFileNameWithoutExtension(messagePath) + ".out";
var txPath = Path.Combine(transactionDir, inProgressFileName);
var committedPath = Path.Combine(commitDir, inProgressFileName);
outgoingFiles.Enqueue(new OutgoingFile(committedPath, messagePath));
return(AsyncFile.WriteText(txPath, messageContents));
}
static Func <string, Task> BuildDefaultDiagnosticsWriter(HostingComponent.Configuration configuration)
{
var diagnosticsRootPath = configuration.DiagnosticsPath;
if (diagnosticsRootPath == null)
{
try
{
diagnosticsRootPath = Path.Combine(Host.GetOutputDirectory(), ".diagnostics");
}
catch (Exception e)
{
logger.Warn("Unable to determine the diagnostics output directory. Check the attached exception for further information, or configure a custom diagnostics directory using 'EndpointConfiguration.SetDiagnosticsPath()'.", e);
return(data => Task.CompletedTask);
}
}
if (!Directory.Exists(diagnosticsRootPath))
{
try
{
Directory.CreateDirectory(diagnosticsRootPath);
}
catch (Exception e)
{
logger.Warn("Unable to create the diagnostics output directory. Check the attached exception for further information, or change the diagnostics directory using 'EndpointConfiguration.SetDiagnosticsPath()'.", e);
return(data => Task.CompletedTask);
}
}
// Once we have the proper hosting model in place we can skip the endpoint name since the host would
// know how to handle multi hosting but for now we do this so that multi-hosting users will get a file per endpoint
var startupDiagnosticsFileName = $"{configuration.EndpointName}-configuration.txt";
var startupDiagnosticsFilePath = Path.Combine(diagnosticsRootPath, startupDiagnosticsFileName);
return(data =>
{
var prettied = JsonPrettyPrinter.Print(data);
return AsyncFile.WriteText(startupDiagnosticsFilePath, prettied);
});
}
static Func <string, Task> BuildDefaultDiagnosticsWriter(ReadOnlySettings settings)
{
if (!settings.TryGet <string>(DiagnosticSettingsExtensions.DiagnosticsPathKey, out var diagnosticsRootPath))
{
diagnosticsRootPath = Path.Combine(Host.GetOutputDirectory(), ".diagnostics");
}
if (!Directory.Exists(diagnosticsRootPath))
{
Directory.CreateDirectory(diagnosticsRootPath);
}
var endpointName = settings.EndpointName();
// Once we have the proper hosting model in place we can skip the endpoint name since the host would
// know how to handle multi hosting but for now we do this so that multi-hosting users will get a file per endpoint
var startupDiagnosticsFileName = $"{endpointName}-configuration.txt";
var startupDiagnosticsFilePath = Path.Combine(diagnosticsRootPath, startupDiagnosticsFileName);
return(data => AsyncFile.WriteText(startupDiagnosticsFilePath, data));
}
async Task ProcessFile(ILearningTransportTransaction transaction, string messageId, CancellationToken messageProcessingCancellationToken)
{
var message = await AsyncFile.ReadText(transaction.FileToProcess, messageProcessingCancellationToken)
.ConfigureAwait(false);
var bodyPath = Path.Combine(bodyDir, $"{messageId}{BodyFileSuffix}");
var headers = HeaderSerializer.Deserialize(message);
if (headers.TryGetValue(LearningTransportHeaders.TimeToBeReceived, out var ttbrString))
{
headers.Remove(LearningTransportHeaders.TimeToBeReceived);
var ttbr = TimeSpan.Parse(ttbrString);
//file.move preserves create time
var sentTime = File.GetCreationTimeUtc(transaction.FileToProcess);
var utcNow = DateTime.UtcNow;
if (sentTime + ttbr < utcNow)
{
await transaction.Commit(messageProcessingCancellationToken)
.ConfigureAwait(false);
log.InfoFormat("Dropping message '{0}' as the specified TimeToBeReceived of '{1}' expired since sending the message at '{2:O}'. Current UTC time is '{3:O}'", messageId, ttbrString, sentTime, utcNow);
return;
}
}
var body = await AsyncFile.ReadBytes(bodyPath, messageProcessingCancellationToken)
.ConfigureAwait(false);
var transportTransaction = new TransportTransaction();
if (transactionMode == TransportTransactionMode.SendsAtomicWithReceive)
{
transportTransaction.Set(transaction);
}
var processingContext = new ContextBag();
var messageContext = new MessageContext(messageId, headers, body, transportTransaction, processingContext);
try
{
await onMessage(messageContext, messageProcessingCancellationToken)
.ConfigureAwait(false);
}
catch (OperationCanceledException ex) when(messageProcessingCancellationToken.IsCancellationRequested)
{
log.Info("Message processing cancelled. Rolling back transaction.", ex);
transaction.Rollback();
return;
}
catch (Exception exception)
{
transaction.ClearPendingOutgoingOperations();
var processingFailures = retryCounts.AddOrUpdate(messageId, id => 1, (id, currentCount) => currentCount + 1);
headers = HeaderSerializer.Deserialize(message);
headers.Remove(LearningTransportHeaders.TimeToBeReceived);
var errorContext = new ErrorContext(exception, headers, messageId, body, transportTransaction, processingFailures, processingContext);
ErrorHandleResult result;
try
{
result = await onError(errorContext, messageProcessingCancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException ex) when(messageProcessingCancellationToken.IsCancellationRequested)
{
log.Info("Message processing cancelled. Rolling back transaction.", ex);
transaction.Rollback();
return;
}
catch (Exception ex)
{
criticalErrorAction($"Failed to execute recoverability policy for message with native ID: `{messageContext.NativeMessageId}`", ex, messageProcessingCancellationToken);
result = ErrorHandleResult.RetryRequired;
}
if (result == ErrorHandleResult.RetryRequired)
{
transaction.Rollback();
return;
}
}
await transaction.Commit(messageProcessingCancellationToken).ConfigureAwait(false);
}
async Task WriteMessage(string destination, IOutgoingTransportOperation transportOperation, TransportTransaction transaction)
{
var message = transportOperation.Message;
var headerPayload = HeaderSerializer.Serialize(message.Headers);
var headerSize = Encoding.UTF8.GetByteCount(headerPayload);
if (headerSize + message.Body.Length > maxMessageSizeKB * 1024)
{
throw new Exception($"The total size of the '{message.Headers[Headers.EnclosedMessageTypes]}' message body ({message.Body.Length} bytes) plus headers ({headerSize} bytes) is larger than {maxMessageSizeKB} KB and will not be supported on some production transports. Consider using the NServiceBus DataBus or the claim check pattern to avoid messages with a large payload. Use 'EndpointConfiguration.UseTransport<LearningTransport>().NoPayloadSizeRestriction()' to disable this check and proceed with the current message size.");
}
var nativeMessageId = Guid.NewGuid().ToString();
var destinationPath = Path.Combine(basePath, destination);
var bodyDir = Path.Combine(destinationPath, LearningTransportMessagePump.BodyDirName);
Directory.CreateDirectory(bodyDir);
var bodyPath = Path.Combine(bodyDir, nativeMessageId) + LearningTransportMessagePump.BodyFileSuffix;
await AsyncFile.WriteBytes(bodyPath, message.Body)
.ConfigureAwait(false);
DateTime?timeToDeliver = null;
if (transportOperation.DeliveryConstraints.TryGet(out DoNotDeliverBefore doNotDeliverBefore))
{
timeToDeliver = doNotDeliverBefore.At;
}
else if (transportOperation.DeliveryConstraints.TryGet(out DelayDeliveryWith delayDeliveryWith))
{
timeToDeliver = DateTime.UtcNow + delayDeliveryWith.Delay;
}
if (timeToDeliver.HasValue)
{
if (transportOperation.DeliveryConstraints.TryGet(out DiscardIfNotReceivedBefore timeToBeReceived) && timeToBeReceived.MaxTime < TimeSpan.MaxValue)
{
throw new Exception($"Postponed delivery of messages with TimeToBeReceived set is not supported. Remove the TimeToBeReceived attribute to postpone messages of type '{message.Headers[Headers.EnclosedMessageTypes]}'.");
}
// we need to "ceil" the seconds to guarantee that we delay with at least the requested value
// since the folder name has only second resolution.
if (timeToDeliver.Value.Millisecond > 0)
{
timeToDeliver += TimeSpan.FromSeconds(1);
}
destinationPath = Path.Combine(destinationPath, LearningTransportMessagePump.DelayedDirName, timeToDeliver.Value.ToString("yyyyMMddHHmmss"));
Directory.CreateDirectory(destinationPath);
}
var messagePath = Path.Combine(destinationPath, nativeMessageId) + ".metadata.txt";
if (transportOperation.RequiredDispatchConsistency != DispatchConsistency.Isolated && transaction.TryGet(out ILearningTransportTransaction directoryBasedTransaction))
{
await directoryBasedTransaction.Enlist(messagePath, headerPayload)
.ConfigureAwait(false);
}
else
{
// atomic avoids the file being locked when the receiver tries to process it
await AsyncFile.WriteTextAtomic(messagePath, headerPayload)
.ConfigureAwait(false);
}
}
public Task Enlist(string messagePath, string messageContents) => AsyncFile.WriteText(messagePath, messageContents);
public Task Enlist(string messagePath, string messageContents, CancellationToken cancellationToken) => AsyncFile.WriteText(messagePath, messageContents, cancellationToken);
async Task ProcessFile(ILearningTransportTransaction transaction, string messageId)
{
var message = await AsyncFile.ReadText(transaction.FileToProcess)
.ConfigureAwait(false);
var bodyPath = Path.Combine(bodyDir, $"{messageId}{BodyFileSuffix}");
var headers = HeaderSerializer.Deserialize(message);
if (headers.TryGetValue(LearningTransportHeaders.TimeToBeReceived, out var ttbrString))
{
headers.Remove(LearningTransportHeaders.TimeToBeReceived);
var ttbr = TimeSpan.Parse(ttbrString);
//file.move preserves create time
var sentTime = File.GetCreationTimeUtc(transaction.FileToProcess);
var utcNow = DateTime.UtcNow;
if (sentTime + ttbr < utcNow)
{
await transaction.Commit()
.ConfigureAwait(false);
log.InfoFormat("Dropping message '{0}' as the specified TimeToBeReceived of '{1}' expired since sending the message at '{2:O}'. Current UTC time is '{3:O}'", messageId, ttbrString, sentTime, utcNow);
return;
}
}
var tokenSource = new CancellationTokenSource();
var body = await AsyncFile.ReadBytes(bodyPath, cancellationToken)
.ConfigureAwait(false);
var transportTransaction = new TransportTransaction();
if (transactionMode == TransportTransactionMode.SendsAtomicWithReceive)
{
transportTransaction.Set(transaction);
}
var messageContext = new MessageContext(messageId, headers, body, transportTransaction, tokenSource, new ContextBag());
try
{
await onMessage(messageContext)
.ConfigureAwait(false);
}
catch (Exception exception)
{
transaction.ClearPendingOutgoingOperations();
var processingFailures = retryCounts.AddOrUpdate(messageId, id => 1, (id, currentCount) => currentCount + 1);
var errorContext = new ErrorContext(exception, headers, messageId, body, transportTransaction, processingFailures);
// the transport tests assume that all transports use a circuit breaker to be resilient against exceptions
// in onError. Since we don't need that robustness, we just retry onError once should it fail.
ErrorHandleResult actionToTake;
try
{
actionToTake = await onError(errorContext)
.ConfigureAwait(false);
}
catch (Exception)
{
actionToTake = await onError(errorContext)
.ConfigureAwait(false);
}
if (actionToTake == ErrorHandleResult.RetryRequired)
{
transaction.Rollback();
return;
}
}
if (tokenSource.IsCancellationRequested)
{
transaction.Rollback();
return;
}
await transaction.Commit()
.ConfigureAwait(false);
}