public Task <bool> BeginTransaction(string incomingFilePath)
        {
            Directory.CreateDirectory(transactionDir);
            FileToProcess = Path.Combine(transactionDir, Path.GetFileName(incomingFilePath));

            return(AsyncFile.Move(incomingFilePath, FileToProcess));
        }
Esempio n. 2
0
        public Task <bool> BeginTransaction(string incomingFilePath, CancellationToken cancellationToken)
        {
            Directory.CreateDirectory(processingDirectory);
            FileToProcess = Path.Combine(processingDirectory, Path.GetFileName(incomingFilePath));

            return(AsyncFile.Move(incomingFilePath, FileToProcess, cancellationToken));
        }
Esempio n. 3
0
        async Task <IEnumerable <string> > GetSubscribersFor(Type messageType)
        {
            var subscribers = new HashSet <string>();

            var allEventTypes = GetPotentialEventTypes(messageType);

            foreach (var eventType in allEventTypes)
            {
                var eventDir = Path.Combine(basePath, ".events", eventType.FullName);

                if (!Directory.Exists(eventDir))
                {
                    continue;
                }

                foreach (var file in Directory.GetFiles(eventDir))
                {
                    var allText = await AsyncFile.ReadText(file)
                                  .ConfigureAwait(false);

                    subscribers.Add(allText);
                }
            }

            return(subscribers);
        }
        async Task Subscribe(MessageMetadata eventType, CancellationToken cancellationToken)
        {
            var eventDir = GetEventDirectory(eventType.MessageType);

            // the subscription directory and the subscription information will be created no matter if there's a publisher for the event assuming that the publisher haven’t started yet
            Directory.CreateDirectory(eventDir);

            var subscriptionEntryPath = GetSubscriptionEntryPath(eventDir);

            var attempts = 0;

            // since we have a design that can run into concurrency exceptions we perform a few retries
            while (true)
            {
                try
                {
                    await AsyncFile.WriteText(subscriptionEntryPath, localAddress, cancellationToken).ConfigureAwait(false);

                    return;
                }
                catch (IOException)
                {
                    attempts++;

                    if (attempts > 10)
                    {
                        throw;
                    }

                    //allow the other task to complete
                    await Task.Delay(100, cancellationToken).ConfigureAwait(false);
                }
            }
        }
        public async Task Subscribe(Type eventType, ContextBag context)
        {
            var eventDir = GetEventDirectory(eventType);

            // that way we can detect that there is indeed a publisher for the event. That said it also means that we will have do "retries" here due to race condition.
            Directory.CreateDirectory(eventDir);

            var subscriptionEntryPath = GetSubscriptionEntryPath(eventDir);

            var attempts = 0;

            // since we have a design that can run into concurrency exceptions we perform a few retries
            while (true)
            {
                try
                {
                    await AsyncFile.WriteText(subscriptionEntryPath, localAddress).ConfigureAwait(false);

                    return;
                }
                catch (IOException)
                {
                    attempts++;

                    if (attempts > 10)
                    {
                        throw;
                    }

                    //allow the other task to complete
                    await Task.Delay(100).ConfigureAwait(false);
                }
            }
        }
        static Func <string, Task> BuildDefaultDiagnosticsWriter(ReadOnlySettings settings)
        {
            if (!settings.TryGet <string>(DiagnosticSettingsExtensions.DiagnosticsPathKey, out var diagnosticsRootPath))
            {
                try
                {
                    diagnosticsRootPath = Path.Combine(Host.GetOutputDirectory(), ".diagnostics");
                }
                catch (Exception e)
                {
                    logger.Error("Unable to determine the diagnostics output directory. Check the attached exception for further information, or configure a custom diagnostics directory using 'EndpointConfiguration.SetDiagnosticsPath()'.", e);
                }
            }

            if (!Directory.Exists(diagnosticsRootPath))
            {
                Directory.CreateDirectory(diagnosticsRootPath);
            }

            var endpointName = settings.EndpointName();

            // Once we have the proper hosting model in place we can skip the endpoint name since the host would
            // know how to handle multi hosting but for now we do this so that multi-hosting users will get a file per endpoint
            var startupDiagnosticsFileName = $"{endpointName}-configuration.txt";
            var startupDiagnosticsFilePath = Path.Combine(diagnosticsRootPath, startupDiagnosticsFileName);

            return(data => AsyncFile.WriteText(startupDiagnosticsFilePath, data));
        }
        public Task Enlist(string messagePath, string messageContents)
        {
            var inProgressFileName = Path.GetFileNameWithoutExtension(messagePath) + ".out";

            var txPath        = Path.Combine(transactionDir, inProgressFileName);
            var committedPath = Path.Combine(commitDir, inProgressFileName);

            outgoingFiles.Enqueue(new OutgoingFile(committedPath, messagePath));

            return(AsyncFile.WriteText(txPath, messageContents));
        }
        static Func <string, Task> BuildDefaultDiagnosticsWriter(HostingComponent.Configuration configuration)
        {
            var diagnosticsRootPath = configuration.DiagnosticsPath;

            if (diagnosticsRootPath == null)
            {
                try
                {
                    diagnosticsRootPath = Path.Combine(Host.GetOutputDirectory(), ".diagnostics");
                }
                catch (Exception e)
                {
                    logger.Warn("Unable to determine the diagnostics output directory. Check the attached exception for further information, or configure a custom diagnostics directory using 'EndpointConfiguration.SetDiagnosticsPath()'.", e);

                    return(data => Task.CompletedTask);
                }
            }

            if (!Directory.Exists(diagnosticsRootPath))
            {
                try
                {
                    Directory.CreateDirectory(diagnosticsRootPath);
                }
                catch (Exception e)
                {
                    logger.Warn("Unable to create the diagnostics output directory. Check the attached exception for further information, or change the diagnostics directory using 'EndpointConfiguration.SetDiagnosticsPath()'.", e);

                    return(data => Task.CompletedTask);
                }
            }

            // Once we have the proper hosting model in place we can skip the endpoint name since the host would
            // know how to handle multi hosting but for now we do this so that multi-hosting users will get a file per endpoint
            var startupDiagnosticsFileName = $"{configuration.EndpointName}-configuration.txt";
            var startupDiagnosticsFilePath = Path.Combine(diagnosticsRootPath, startupDiagnosticsFileName);


            return(data =>
            {
                var prettied = JsonPrettyPrinter.Print(data);
                return AsyncFile.WriteText(startupDiagnosticsFilePath, prettied);
            });
        }
        static Func <string, Task> BuildDefaultDiagnosticsWriter(ReadOnlySettings settings)
        {
            if (!settings.TryGet <string>(DiagnosticSettingsExtensions.DiagnosticsPathKey, out var diagnosticsRootPath))
            {
                diagnosticsRootPath = Path.Combine(Host.GetOutputDirectory(), ".diagnostics");
            }

            if (!Directory.Exists(diagnosticsRootPath))
            {
                Directory.CreateDirectory(diagnosticsRootPath);
            }

            var endpointName = settings.EndpointName();

            // Once we have the proper hosting model in place we can skip the endpoint name since the host would
            // know how to handle multi hosting but for now we do this so that multi-hosting users will get a file per endpoint
            var startupDiagnosticsFileName = $"{endpointName}-configuration.txt";
            var startupDiagnosticsFilePath = Path.Combine(diagnosticsRootPath, startupDiagnosticsFileName);

            return(data => AsyncFile.WriteText(startupDiagnosticsFilePath, data));
        }
        async Task ProcessFile(ILearningTransportTransaction transaction, string messageId, CancellationToken messageProcessingCancellationToken)
        {
            var message = await AsyncFile.ReadText(transaction.FileToProcess, messageProcessingCancellationToken)
                          .ConfigureAwait(false);

            var bodyPath = Path.Combine(bodyDir, $"{messageId}{BodyFileSuffix}");
            var headers  = HeaderSerializer.Deserialize(message);

            if (headers.TryGetValue(LearningTransportHeaders.TimeToBeReceived, out var ttbrString))
            {
                headers.Remove(LearningTransportHeaders.TimeToBeReceived);

                var ttbr = TimeSpan.Parse(ttbrString);

                //file.move preserves create time
                var sentTime = File.GetCreationTimeUtc(transaction.FileToProcess);

                var utcNow = DateTime.UtcNow;
                if (sentTime + ttbr < utcNow)
                {
                    await transaction.Commit(messageProcessingCancellationToken)
                    .ConfigureAwait(false);

                    log.InfoFormat("Dropping message '{0}' as the specified TimeToBeReceived of '{1}' expired since sending the message at '{2:O}'. Current UTC time is '{3:O}'", messageId, ttbrString, sentTime, utcNow);
                    return;
                }
            }

            var body = await AsyncFile.ReadBytes(bodyPath, messageProcessingCancellationToken)
                       .ConfigureAwait(false);

            var transportTransaction = new TransportTransaction();

            if (transactionMode == TransportTransactionMode.SendsAtomicWithReceive)
            {
                transportTransaction.Set(transaction);
            }

            var processingContext = new ContextBag();

            var messageContext = new MessageContext(messageId, headers, body, transportTransaction, processingContext);

            try
            {
                await onMessage(messageContext, messageProcessingCancellationToken)
                .ConfigureAwait(false);
            }
            catch (OperationCanceledException ex) when(messageProcessingCancellationToken.IsCancellationRequested)
            {
                log.Info("Message processing cancelled. Rolling back transaction.", ex);
                transaction.Rollback();

                return;
            }
            catch (Exception exception)
            {
                transaction.ClearPendingOutgoingOperations();

                var processingFailures = retryCounts.AddOrUpdate(messageId, id => 1, (id, currentCount) => currentCount + 1);

                headers = HeaderSerializer.Deserialize(message);
                headers.Remove(LearningTransportHeaders.TimeToBeReceived);

                var errorContext = new ErrorContext(exception, headers, messageId, body, transportTransaction, processingFailures, processingContext);

                ErrorHandleResult result;
                try
                {
                    result = await onError(errorContext, messageProcessingCancellationToken).ConfigureAwait(false);
                }
                catch (OperationCanceledException ex) when(messageProcessingCancellationToken.IsCancellationRequested)
                {
                    log.Info("Message processing cancelled. Rolling back transaction.", ex);
                    transaction.Rollback();

                    return;
                }
                catch (Exception ex)
                {
                    criticalErrorAction($"Failed to execute recoverability policy for message with native ID: `{messageContext.NativeMessageId}`", ex, messageProcessingCancellationToken);
                    result = ErrorHandleResult.RetryRequired;
                }

                if (result == ErrorHandleResult.RetryRequired)
                {
                    transaction.Rollback();

                    return;
                }
            }

            await transaction.Commit(messageProcessingCancellationToken).ConfigureAwait(false);
        }
Esempio n. 11
0
        async Task WriteMessage(string destination, IOutgoingTransportOperation transportOperation, TransportTransaction transaction)
        {
            var message       = transportOperation.Message;
            var headerPayload = HeaderSerializer.Serialize(message.Headers);
            var headerSize    = Encoding.UTF8.GetByteCount(headerPayload);

            if (headerSize + message.Body.Length > maxMessageSizeKB * 1024)
            {
                throw new Exception($"The total size of the '{message.Headers[Headers.EnclosedMessageTypes]}' message body ({message.Body.Length} bytes) plus headers ({headerSize} bytes) is larger than {maxMessageSizeKB} KB and will not be supported on some production transports. Consider using the NServiceBus DataBus or the claim check pattern to avoid messages with a large payload. Use 'EndpointConfiguration.UseTransport<LearningTransport>().NoPayloadSizeRestriction()' to disable this check and proceed with the current message size.");
            }

            var nativeMessageId = Guid.NewGuid().ToString();
            var destinationPath = Path.Combine(basePath, destination);
            var bodyDir         = Path.Combine(destinationPath, LearningTransportMessagePump.BodyDirName);

            Directory.CreateDirectory(bodyDir);

            var bodyPath = Path.Combine(bodyDir, nativeMessageId) + LearningTransportMessagePump.BodyFileSuffix;

            await AsyncFile.WriteBytes(bodyPath, message.Body)
            .ConfigureAwait(false);

            DateTime?timeToDeliver = null;

            if (transportOperation.DeliveryConstraints.TryGet(out DoNotDeliverBefore doNotDeliverBefore))
            {
                timeToDeliver = doNotDeliverBefore.At;
            }
            else if (transportOperation.DeliveryConstraints.TryGet(out DelayDeliveryWith delayDeliveryWith))
            {
                timeToDeliver = DateTime.UtcNow + delayDeliveryWith.Delay;
            }

            if (timeToDeliver.HasValue)
            {
                if (transportOperation.DeliveryConstraints.TryGet(out DiscardIfNotReceivedBefore timeToBeReceived) && timeToBeReceived.MaxTime < TimeSpan.MaxValue)
                {
                    throw new Exception($"Postponed delivery of messages with TimeToBeReceived set is not supported. Remove the TimeToBeReceived attribute to postpone messages of type '{message.Headers[Headers.EnclosedMessageTypes]}'.");
                }

                // we need to "ceil" the seconds to guarantee that we delay with at least the requested value
                // since the folder name has only second resolution.
                if (timeToDeliver.Value.Millisecond > 0)
                {
                    timeToDeliver += TimeSpan.FromSeconds(1);
                }

                destinationPath = Path.Combine(destinationPath, LearningTransportMessagePump.DelayedDirName, timeToDeliver.Value.ToString("yyyyMMddHHmmss"));

                Directory.CreateDirectory(destinationPath);
            }

            var messagePath = Path.Combine(destinationPath, nativeMessageId) + ".metadata.txt";

            if (transportOperation.RequiredDispatchConsistency != DispatchConsistency.Isolated && transaction.TryGet(out ILearningTransportTransaction directoryBasedTransaction))
            {
                await directoryBasedTransaction.Enlist(messagePath, headerPayload)
                .ConfigureAwait(false);
            }
            else
            {
                // atomic avoids the file being locked when the receiver tries to process it
                await AsyncFile.WriteTextAtomic(messagePath, headerPayload)
                .ConfigureAwait(false);
            }
        }
 public Task Enlist(string messagePath, string messageContents) => AsyncFile.WriteText(messagePath, messageContents);
Esempio n. 13
0
 public Task Enlist(string messagePath, string messageContents, CancellationToken cancellationToken) => AsyncFile.WriteText(messagePath, messageContents, cancellationToken);
        async Task ProcessFile(ILearningTransportTransaction transaction, string messageId)
        {
            var message = await AsyncFile.ReadText(transaction.FileToProcess)
                          .ConfigureAwait(false);

            var bodyPath = Path.Combine(bodyDir, $"{messageId}{BodyFileSuffix}");
            var headers  = HeaderSerializer.Deserialize(message);

            if (headers.TryGetValue(LearningTransportHeaders.TimeToBeReceived, out var ttbrString))
            {
                headers.Remove(LearningTransportHeaders.TimeToBeReceived);

                var ttbr = TimeSpan.Parse(ttbrString);

                //file.move preserves create time
                var sentTime = File.GetCreationTimeUtc(transaction.FileToProcess);

                var utcNow = DateTime.UtcNow;
                if (sentTime + ttbr < utcNow)
                {
                    await transaction.Commit()
                    .ConfigureAwait(false);

                    log.InfoFormat("Dropping message '{0}' as the specified TimeToBeReceived of '{1}' expired since sending the message at '{2:O}'. Current UTC time is '{3:O}'", messageId, ttbrString, sentTime, utcNow);
                    return;
                }
            }

            var tokenSource = new CancellationTokenSource();

            var body = await AsyncFile.ReadBytes(bodyPath, cancellationToken)
                       .ConfigureAwait(false);

            var transportTransaction = new TransportTransaction();

            if (transactionMode == TransportTransactionMode.SendsAtomicWithReceive)
            {
                transportTransaction.Set(transaction);
            }

            var messageContext = new MessageContext(messageId, headers, body, transportTransaction, tokenSource, new ContextBag());

            try
            {
                await onMessage(messageContext)
                .ConfigureAwait(false);
            }
            catch (Exception exception)
            {
                transaction.ClearPendingOutgoingOperations();
                var processingFailures = retryCounts.AddOrUpdate(messageId, id => 1, (id, currentCount) => currentCount + 1);

                var errorContext = new ErrorContext(exception, headers, messageId, body, transportTransaction, processingFailures);

                // the transport tests assume that all transports use a circuit breaker to be resilient against exceptions
                // in onError. Since we don't need that robustness, we just retry onError once should it fail.
                ErrorHandleResult actionToTake;
                try
                {
                    actionToTake = await onError(errorContext)
                                   .ConfigureAwait(false);
                }
                catch (Exception)
                {
                    actionToTake = await onError(errorContext)
                                   .ConfigureAwait(false);
                }

                if (actionToTake == ErrorHandleResult.RetryRequired)
                {
                    transaction.Rollback();

                    return;
                }
            }

            if (tokenSource.IsCancellationRequested)
            {
                transaction.Rollback();

                return;
            }

            await transaction.Commit()
            .ConfigureAwait(false);
        }