public async Task ChangeConcurrency(PushRuntimeSettings newLimitations, CancellationToken cancellationToken = default)
{
await StopReceive(cancellationToken).ConfigureAwait(false);
limitations = newLimitations;
await StartReceive(cancellationToken).ConfigureAwait(false);
}
public void Start(PushRuntimeSettings limitations)
{
if (throwCritical)
{
criticalError.Raise(exceptionToThrow.Message, exceptionToThrow);
}
}
public void Start(PushRuntimeSettings limitations)
{
var definition = topologySectionManager.DetermineReceiveResources(inputQueue);
throttler = new SemaphoreSlim(limitations.MaxConcurrency);
topologyOperator.Start(definition, limitations.MaxConcurrency);
}
public async Task Initialize(PushRuntimeSettings limitations, OnMessage onMessage, OnError onError, CancellationToken cancellationToken = default)
{
this.limitations = limitations;
messageReceivingCancellationTokenSource = new CancellationTokenSource();
messageProcessingCancellationTokenSource = new CancellationTokenSource();
processStrategy = processStrategyFactory(transport.TransportTransactionMode);
messageReceivingCircuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("message receiving", waitTimeCircuitBreaker, ex => hostSettings.CriticalErrorAction("Failed to peek " + receiveSettings.ReceiveAddress, ex, messageProcessingCancellationTokenSource.Token));
messageProcessingCircuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("message processing", waitTimeCircuitBreaker, ex => hostSettings.CriticalErrorAction("Failed to receive from " + receiveSettings.ReceiveAddress, ex, messageProcessingCancellationTokenSource.Token));
inputQueue = queueFactory(receiveSettings.ReceiveAddress);
errorQueue = queueFactory(receiveSettings.ErrorQueue);
processStrategy.Init(inputQueue, errorQueue, onMessage, onError, hostSettings.CriticalErrorAction);
if (transport.ExpiredMessagesPurger.PurgeOnStartup)
{
try
{
var purgedRowsCount = await queuePurger.Purge(inputQueue, cancellationToken).ConfigureAwait(false);
Logger.InfoFormat("{0:N0} messages purged from queue {1}", purgedRowsCount, receiveSettings.ReceiveAddress);
}
catch (Exception ex) when(!ex.IsCausedBy(cancellationToken))
{
Logger.Warn("Failed to purge input queue on startup.", ex);
}
}
await PurgeExpiredMessages(cancellationToken).ConfigureAwait(false);
await schemaInspector.PerformInspection(inputQueue, cancellationToken).ConfigureAwait(false);
}
public void Start(PushRuntimeSettings limitations)
{
if (throwCritical)
{
criticalError.Raise(exceptionToThrow.Message, exceptionToThrow);
}
}
public void Start(PushRuntimeSettings limitations)
{
if (throwCritical != null)
{
criticalError.Raise(throwCritical.Message, throwCritical);
}
}
public Task ChangeConcurrency(PushRuntimeSettings limitations, CancellationToken cancellationToken = default)
{
maxConcurrency = limitations.MaxConcurrency;
Logger.InfoFormat("Calling a change concurrency and reconnecting with new value {0}.", limitations.MaxConcurrency);
_ = Task.Run(() => Reconnect(), cancellationToken);
return(Task.CompletedTask);
}
public Configuration(LogicalAddress logicalAddress,
string queueNameBase,
string localAddress,
string instanceSpecificQueue,
TransportTransactionMode transactionMode,
PushRuntimeSettings pushRuntimeSettings,
bool purgeOnStartup,
Notification <ReceivePipelineCompleted> pipelineCompletedSubscribers,
bool isSendOnlyEndpoint,
List <Type> executeTheseHandlersFirst,
MessageHandlerRegistry messageHandlerRegistry,
bool createQueues, TransportSeam transportSeam)
{
LogicalAddress = logicalAddress;
QueueNameBase = queueNameBase;
LocalAddress = localAddress;
InstanceSpecificQueue = instanceSpecificQueue;
TransactionMode = transactionMode;
PushRuntimeSettings = pushRuntimeSettings;
PurgeOnStartup = purgeOnStartup;
IsSendOnlyEndpoint = isSendOnlyEndpoint;
PipelineCompletedSubscribers = pipelineCompletedSubscribers;
ExecuteTheseHandlersFirst = executeTheseHandlersFirst;
satelliteDefinitions = new List <SatelliteDefinition>();
this.messageHandlerRegistry = messageHandlerRegistry;
CreateQueues = createQueues;
this.transportSeam = transportSeam;
}
public Task Initialize(PushRuntimeSettings limitations, OnMessage onMessage, OnError onError, CancellationToken cancellationToken = default)
{
var inputAddress = MsmqAddress.Parse(receiveSettings.ReceiveAddress);
var errorAddress = MsmqAddress.Parse(receiveSettings.ErrorQueue);
if (!string.Equals(inputAddress.Machine, RuntimeEnvironment.MachineName,
StringComparison.OrdinalIgnoreCase))
{
throw new Exception(
$"MSMQ Dequeuing can only run against the local machine. Invalid inputQueue name '{receiveSettings.ReceiveAddress}'.");
}
inputQueue = new MessageQueue(inputAddress.FullPath, false, true, QueueAccessMode.Receive);
errorQueue = new MessageQueue(errorAddress.FullPath, false, true, QueueAccessMode.Send);
if (transportSettings.TransportTransactionMode != TransportTransactionMode.None && !QueueIsTransactional())
{
throw new ArgumentException(
$"Queue must be transactional if you configure the endpoint to be transactional ({receiveSettings.ReceiveAddress}).");
}
inputQueue.MessageReadPropertyFilter = DefaultReadPropertyFilter;
if (receiveSettings.PurgeOnStartup)
{
inputQueue.Purge();
}
receiveStrategy = receiveStrategyFactory(transportSettings.TransportTransactionMode);
receiveStrategy.Init(inputQueue, errorQueue, onMessage, onError, criticalErrorAction, transportSettings.IgnoreIncomingTimeToBeReceivedHeaders);
maxConcurrency = limitations.MaxConcurrency;
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency, limitations.MaxConcurrency);
return(Task.CompletedTask);
}
public void Start(PushRuntimeSettings limitations)
{
maxConcurrency = limitations.MaxConcurrency;
var prefetchCount = maxConcurrency * prefetchMultiplier;
if (overriddenPrefetchCount.HasValue)
{
prefetchCount = overriddenPrefetchCount.Value;
}
var receiveMode = pushSettings.RequiredTransactionMode == TransportTransactionMode.None ? ReceiveMode.ReceiveAndDelete : ReceiveMode.PeekLock;
if (tokenProvider == null)
{
receiver = new MessageReceiver(connectionStringBuilder.GetNamespaceConnectionString(), pushSettings.InputQueue, receiveMode, retryPolicy: retryPolicy, prefetchCount);
}
else
{
receiver = new MessageReceiver(connectionStringBuilder.Endpoint, pushSettings.InputQueue, tokenProvider, connectionStringBuilder.TransportType, receiveMode, retryPolicy: retryPolicy, prefetchCount);
}
semaphore = new SemaphoreSlim(maxConcurrency, maxConcurrency);
messageProcessing = new CancellationTokenSource();
receiveLoopTask = Task.Run(() => ReceiveLoop());
}
private void HydrateFields(PushRuntimeSettings pushRuntimeSettings)
{
_runningReceiveTasks = new ConcurrentDictionary <Task, Task>();
_concurrencyLimiter = new SemaphoreSlim(pushRuntimeSettings.MaxConcurrency);
_cancellationTokenSource = new CancellationTokenSource();
_cancellationToken = _cancellationTokenSource.Token;
}
public SatelliteDefinition(string name, string receiveAddress, PushRuntimeSettings runtimeSettings, Func <RecoverabilityConfig, ErrorContext, RecoverabilityAction> recoverabilityPolicy, OnSatelliteMessage onMessage)
{
Name = name;
ReceiveAddress = receiveAddress;
RuntimeSettings = runtimeSettings;
RecoverabilityPolicy = recoverabilityPolicy;
OnMessage = onMessage;
}
public void Start(PushRuntimeSettings limitations)
{
// The LimitMessageProcessingConcurrencyTo setting only applies to the input queue
if (pushSettings.InputQueue == Conventions.EndpointNamingConvention(typeof(ThrottledEndpoint)))
{
Assert.AreEqual(10, limitations.MaxConcurrency);
}
}
public void Start(PushRuntimeSettings limitations)
{
logger.Info($"Starting MessagePump for {this.messagePumpName}");
this.queue = session.GetQueue(this.nsbSettings.InputQueue);
this.consumer = session.CreateConsumer(queue);
this.consumer.Listener += HandleConsumerMessage;
}
public void Start(PushRuntimeSettings limitations)
{
// The LimitMessageProcessingConcurrencyTo setting only applies to the input queue
if (pushSettings.InputQueue == Conventions.EndpointNamingConvention(typeof(ThrottledEndpoint)))
{
Assert.AreEqual(10, limitations.MaxConcurrency);
}
}
public void Start(PushRuntimeSettings limitations)
{
if (ThrowOnStart)
{
throw new InvalidOperationException();
}
Started = true;
}
public SatelliteDefinition(string name, string receiveAddress, TransportTransactionMode requiredTransportTransactionMode, PushRuntimeSettings runtimeSettings, Func <RecoverabilityConfig, ErrorContext, RecoverabilityAction> recoverabilityPolicy, Func <IBuilder, MessageContext, Task> onMessage)
{
Name = name;
ReceiveAddress = receiveAddress;
RequiredTransportTransactionMode = requiredTransportTransactionMode;
RuntimeSettings = runtimeSettings;
RecoverabilityPolicy = recoverabilityPolicy;
OnMessage = onMessage;
}
public SatelliteDefinition(string name, string receiveAddress, TransportTransactionMode requiredTransportTransactionMode, PushRuntimeSettings runtimeSettings, Func<RecoverabilityConfig, ErrorContext, RecoverabilityAction> recoverabilityPolicy, Func<IBuilder, MessageContext, Task> onMessage)
{
Name = name;
ReceiveAddress = receiveAddress;
RequiredTransportTransactionMode = requiredTransportTransactionMode;
RuntimeSettings = runtimeSettings;
RecoverabilityPolicy = recoverabilityPolicy;
OnMessage = onMessage;
}
public void Start(PushRuntimeSettings limitations)
{
cancellationTokenSource = new CancellationTokenSource();
maxConcurrency = limitations.MaxConcurrency;
int numberOfPumps;
if (maxConcurrency <= 10)
{
numberOfPumps = 1;
numberOfMessagesToFetch = maxConcurrency;
}
else
{
numberOfMessagesToFetch = 10;
numberOfPumps = Convert.ToInt32(Math.Ceiling(Convert.ToDouble(maxConcurrency) / numberOfMessagesToFetch));
}
receiveMessagesRequest = new ReceiveMessageRequest
{
MaxNumberOfMessages = numberOfMessagesToFetch,
QueueUrl = queueUrl,
WaitTimeSeconds = 20,
AttributeNames = new List <string> {
"SentTimestamp"
},
MessageAttributeNames = new List <string> {
Headers.MessageId
}
};
maxConcurrencySemaphore = new SemaphoreSlim(maxConcurrency);
pumpTasks = new List <Task>(numberOfPumps);
for (var i = 0; i < numberOfPumps; i++)
{
pumpTasks.Add(Task.Run(() => ConsumeMessages(cancellationTokenSource.Token), CancellationToken.None));
}
if (configuration.IsDelayedDeliveryEnabled)
{
var receiveDelayedMessagesRequest = new ReceiveMessageRequest
{
MaxNumberOfMessages = 10,
QueueUrl = delayedDeliveryQueueUrl,
WaitTimeSeconds = 20,
AttributeNames = new List <string> {
"MessageDeduplicationId", "SentTimestamp", "ApproximateFirstReceiveTimestamp", "ApproximateReceiveCount"
},
MessageAttributeNames = new List <string> {
"All"
}
};
pumpTasks.Add(Task.Run(() => ConsumeDelayedMessages(receiveDelayedMessagesRequest, cancellationTokenSource.Token), CancellationToken.None));
}
}
public void Start(PushRuntimeSettings limitations)
{
settings.Get <FakeTransport.StartUpSequence>().Add($"{nameof(IPushMessages)}.{nameof(Start)}");
if (throwCritical)
{
criticalError.Raise(exceptionToThrow.Message, exceptionToThrow);
}
}
public void Start(PushRuntimeSettings limitations)
{
poller = Task.Run(async() =>
{
using (var repo = new Repository($"../../../../{endpointName}", null))
{
while (!tokenSource.IsCancellationRequested)
{
PullOptions options = new PullOptions
{
FetchOptions = new FetchOptions
{
CredentialsProvider = (url, usernameFromUrl, types) =>
new UsernamePasswordCredentials
{
Username = "******",
Password = "******"
}
}
};
repo.Network.Pull(new Signature("daniel", "daniel", new DateTimeOffset(DateTime.Now)),
options);
foreach (var commit in repo.Commits)
{
if (!alreadyPushedCommits.Contains(commit.Id.Sha))
{
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
{
var headers = commit.Message
.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries)
.Select(
kvp => kvp.Split(new[] { " = " }, StringSplitOptions.RemoveEmptyEntries))
.Select(kvp => new { Key = kvp[0], Value = kvp[1] })
.ToDictionary(x => x.Key, x => x.Value);
var treeEntry = commit.Tree.FirstOrDefault(x => x.Name.EndsWith(".payload"));
if (treeEntry != null)
{
await writer.WriteAsync(File.ReadAllText(treeEntry.Path)).ConfigureAwait(false);
}
await stream.FlushAsync().ConfigureAwait(false);
stream.Position = 0;
var pushContext = new PushContext(commit.Id.Sha, headers, stream,
new NoOpTransaction(), new ContextBag(null));
await pipe(pushContext);
alreadyPushedCommits.Add(commit.Id.Sha);
}
}
}
}
}
});
}
public void Start(PushRuntimeSettings limitations)
{
if (ThrowOnStart)
{
throw new InvalidOperationException();
}
Started = true;
}
public Task Initialize(PushRuntimeSettings limitations, OnMessage onMessage, OnError onError,
CancellationToken cancellationToken)
{
this.onMessage = onMessage;
this.onError = onError;
return(baseTransportReceiver?.Initialize(limitations,
(_, __) => Task.CompletedTask,
(_, __) => Task.FromResult(ErrorHandleResult.Handled),
cancellationToken) ?? Task.CompletedTask);
}
public void Start(PushRuntimeSettings limitations)
{
runningReceiveTasks = new ConcurrentDictionary <Task, Task>();
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency);
cancellationTokenSource = new CancellationTokenSource();
cancellationToken = cancellationTokenSource.Token;
messagePumpTask = Task.Run(ProcessMessages, CancellationToken.None);
}
public void Start(PushRuntimeSettings limitations)
{
runningReceiveTasks = new ConcurrentDictionary <Task, Task>();
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency);
cancellationTokenSource = new CancellationTokenSource();
cancellationToken = cancellationTokenSource.Token;
subscription = connection.ConnectToPersistentSubscription(inputQueue, inputQueue, OnEvent, SubscriptionDropped, autoAck: false);
}
/// <summary>
/// Adds a new satellite receiver.
/// </summary>
/// <param name="name">Name of the satellite.</param>
/// <param name="runtimeSettings">Transport runtime settings.</param>
/// <param name="onMessage">The message func.</param>
/// <param name="transportAddress">The autogenerated transport address to listen on.</param>
/// <param name="recoverabilityPolicy">Recoverability policy to be if processing fails.</param>
public void AddSatelliteReceiver(string name, string transportAddress, PushRuntimeSettings runtimeSettings, Func <RecoverabilityConfig, ErrorContext, RecoverabilityAction> recoverabilityPolicy, Func <IBuilder, MessageContext, Task> onMessage)
{
var requiredTransactionMode = Settings.GetRequiredTransactionModeForReceives();
var satelliteDefinition = new SatelliteDefinition(name, transportAddress, requiredTransactionMode, runtimeSettings, recoverabilityPolicy, onMessage);
Settings.Get <SatelliteDefinitions>().Add(satelliteDefinition);
Settings.Get <QueueBindings>().BindReceiving(transportAddress);
}
public Task Initialize(PushRuntimeSettings limitations, OnMessage onMessage, OnError onError, CancellationToken cancellationToken = default)
{
this.limitations = limitations;
Logger.Debug($"Initializing MessageReceiver {Id}");
receiveStrategy = ReceiveStrategy.BuildReceiveStrategy(onMessage, onError, requiredTransactionMode, criticalErrorAction);
return(azureMessageQueueReceiver.Init(receiveAddress, errorQueue, cancellationToken));
}
public void Start(PushRuntimeSettings limitations)
{
messageProcessing = new CancellationTokenSource();
maxConcurrency = limitations.MaxConcurrency;
semaphore = new SemaphoreSlim(limitations.MaxConcurrency, limitations.MaxConcurrency);
Parallel.ForEach(consumerHolderList, ch => ch.Init(messageProcessing));
Parallel.ForEach(consumerHolderList, ch => ch.Start(limitations));
}
/// <summary>
/// Adds a new satellite receiver.
/// </summary>
/// <param name="name">Name of the satellite.</param>
/// <param name="runtimeSettings">Transport runtime settings.</param>
/// <param name="onMessage">The message func.</param>
/// <param name="transportAddress">The autogenerated transport address to listen on.</param>
/// <param name="recoverabilityPolicy">Recoverability policy to be if processing fails.</param>
public void AddSatelliteReceiver(string name, string transportAddress, PushRuntimeSettings runtimeSettings, Func <RecoverabilityConfig, ErrorContext, RecoverabilityAction> recoverabilityPolicy, Func <IServiceProvider, MessageContext, Task> onMessage)
{
Guard.AgainstNullAndEmpty(nameof(name), name);
Guard.AgainstNullAndEmpty(nameof(transportAddress), transportAddress);
Guard.AgainstNull(nameof(runtimeSettings), runtimeSettings);
Guard.AgainstNull(nameof(recoverabilityPolicy), recoverabilityPolicy);
Guard.AgainstNull(nameof(onMessage), onMessage);
Receiving.AddSatelliteReceiver(name, transportAddress, runtimeSettings, recoverabilityPolicy, onMessage);
}
public void Start(PushRuntimeSettings limitations)
{
inputQueue.FormatPeekCommand(Math.Min(100, 10 * limitations.MaxConcurrency));
runningReceiveTasks = new ConcurrentDictionary <Task, Task>();
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency);
cancellationTokenSource = new CancellationTokenSource();
cancellationToken = cancellationTokenSource.Token;
messagePumpTask = Task.Run(ProcessMessages, CancellationToken.None);
}
public void Start(PushRuntimeSettings limitations)
{
SimpleSubscriber.CreateAsync(subscriptionName)
.ContinueWith(async t =>
{
var s = await t.ConfigureAwait(false);
await s.StartAsync(Consume)
.ConfigureAwait(false);
subscriber = s;
});
}
public void Start(PushRuntimeSettings limitations)
{
inputQueue.FormatPeekCommand(queuePeekerOptions.MaxRecordsToPeek ?? Math.Min(100, 10 * limitations.MaxConcurrency));
maxConcurrency = limitations.MaxConcurrency;
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency);
cancellationTokenSource = new CancellationTokenSource();
cancellationToken = cancellationTokenSource.Token;
messagePumpTask = Task.Run(ProcessMessages, CancellationToken.None);
}
public void Start(PushRuntimeSettings limitations)
{
cancellationTokenSource = new CancellationTokenSource();
inputQueuePump.Start(limitations.MaxConcurrency, cancellationTokenSource.Token);
if (configuration.IsDelayedDeliveryEnabled)
{
delayedMessagesPump.Start(cancellationTokenSource.Token);
}
}
public void Start(PushRuntimeSettings limitations)
{
_cancellationTokenSource = new CancellationTokenSource();
_concurrencyLevel = limitations.MaxConcurrency;
_maxConcurrencySempahore = new SemaphoreSlim(_concurrencyLevel);
_consumerTasks = new List <Task>();
for (var i = 0; i < _concurrencyLevel; i++)
{
_consumerTasks.Add(ConsumeMessages());
}
}
public void Start(PushRuntimeSettings limitations)
{
MessageQueue.ClearConnectionCache();
runningReceiveTasks = new ConcurrentDictionary<Task, Task>();
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency);
cancellationTokenSource = new CancellationTokenSource();
cancellationToken = cancellationTokenSource.Token;
// ReSharper disable once ConvertClosureToMethodGroup
// LongRunning is useless combined with async/await
messagePumpTask = Task.Run(() => ProcessMessages(), CancellationToken.None);
}
public void Start(PushRuntimeSettings limitations)
{
runningReceiveTasks = new ConcurrentDictionary<Task, Task>();
concurrencyLimiter = new SemaphoreSlim(limitations.MaxConcurrency);
cancellationTokenSource = new CancellationTokenSource();
cancellationToken = cancellationTokenSource.Token;
if (purgeOnStartup)
{
Directory.Delete(path, true);
Directory.CreateDirectory(path);
}
messagePumpTask = Task.Factory
.StartNew(ProcessMessages, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)
.Unwrap();
}
public TransportReceiver(
string id,
IPushMessages pushMessages,
PushSettings pushSettings,
PushRuntimeSettings pushRuntimeSettings,
IPipelineExecutor pipelineExecutor,
RecoverabilityExecutor recoverabilityExecutor,
CriticalError criticalError)
{
this.criticalError = criticalError;
Id = id;
this.pushRuntimeSettings = pushRuntimeSettings;
this.pipelineExecutor = pipelineExecutor;
this.recoverabilityExecutor = recoverabilityExecutor;
this.pushSettings = pushSettings;
receiver = pushMessages;
}
static string SetupDispatcherSatellite(FeatureConfigurationContext context, TransportTransactionMode requiredTransactionSupport, PushRuntimeSettings pushRuntimeSettings)
{
var satelliteLogicalAddress = context.Settings.LogicalAddress().CreateQualifiedAddress("TimeoutsDispatcher");
var satelliteAddress = context.Settings.GetTransportAddress(satelliteLogicalAddress);
context.AddSatelliteReceiver("Timeout Dispatcher Processor", satelliteAddress, requiredTransactionSupport, pushRuntimeSettings, RecoverabilityPolicy,
(builder, pushContext) =>
{
var dispatchBehavior = new DispatchTimeoutBehavior(
builder.Build<IDispatchMessages>(),
builder.Build<IPersistTimeouts>(),
requiredTransactionSupport);
return dispatchBehavior.Invoke(pushContext);
});
return satelliteAddress;
}
static void SetupStorageSatellite(FeatureConfigurationContext context, TransportTransactionMode requiredTransactionSupport, PushRuntimeSettings pushRuntimeSettings)
{
var satelliteLogicalAddress = context.Settings.LogicalAddress().CreateQualifiedAddress("Timeouts");
var satelliteAddress = context.Settings.GetTransportAddress(satelliteLogicalAddress);
context.AddSatelliteReceiver("Timeout Message Processor", satelliteAddress, requiredTransactionSupport, pushRuntimeSettings, RecoverabilityPolicy,
(builder, pushContext) =>
{
var storeBehavior = new StoreTimeoutBehavior(
builder.Build<ExpiredTimeoutsPoller>(),
builder.Build<IDispatchMessages>(),
builder.Build<IPersistTimeouts>(),
context.Settings.EndpointName().ToString());
return storeBehavior.Invoke(pushContext);
});
context.Settings.Get<TimeoutManagerAddressConfiguration>().Set(satelliteAddress);
}
public void Start(PushRuntimeSettings limitations)
{
throw new NotImplementedException();
}
