public GZipProcessor(IQueueReader queueReader, IQueueWriter queueWriter, IGZipProgress gZipProgress, IValidator validator)
{
_queueReader = queueReader;
_queueWriter = queueWriter;
_gZipProgress = gZipProgress;
_validator = validator;
}
public void Shutdown(Func <Exception> pendingExceptionGenerator)
{
IQueueReader <T>[] array = null;
lock (this.ThisLock)
{
if ((this.queueState == QueueState <T> .Shutdown) || (this.queueState == QueueState <T> .Closed))
{
return;
}
this.queueState = QueueState <T> .Shutdown;
if ((this.readerQueue.Count > 0) && (this.itemQueue.ItemCount == 0))
{
array = new IQueueReader <T> [this.readerQueue.Count];
this.readerQueue.CopyTo(array, 0);
this.readerQueue.Clear();
}
}
if (array != null)
{
for (int i = 0; i < array.Length; i++)
{
Exception exception = (pendingExceptionGenerator != null) ? pendingExceptionGenerator() : null;
array[i].Set(new Item <T>(exception, null));
}
}
}
internal StaThread(IQueueReader <SendOrPostCallbackItem> reader)
{
mQueueConsumer = reader;
mStaThread = new Thread(Run);
mStaThread.Name = "STA Worker Thread";
mStaThread.SetApartmentState(ApartmentState.STA);
}
public FileReaderForDecompress(IQueueReader queueReader, IGZipProgress gZipProgress)
{
_queueReader = queueReader;
_gZipProgress = gZipProgress;
_fileProcessData = FileProcessData.Instance;
}
public FileDecompressor(IQueueReader queueReader, IQueueWriter queueWriter)
{
_queueReader = queueReader;
_queueWriter = queueWriter;
_fileProcessData = FileProcessData.Instance;
}
public IncomingMessagesStream(IQueueReader[] readers)
{
//check if there are any readers that can be used
if (readers == null) throw new ArgumentNullException("Readers expected");
_readers = readers.Where(x => x != null).ToArray();
if (_readers.Length == 0) throw new ArgumentNullException("Readers expected");
}
public void Handle(IQueueReader inbox, Action <ImmutableEnvelope> lambda, string name = null)
{
var dispatcherName = name ?? "inbox-" + Interlocked.Increment(ref _counter);
var dispatcher = new EnvelopeDispatcher(lambda, Streamer, Quarantine, Duplication, dispatcherName);
AddTask(new DispatcherProcess(dispatcher.Dispatch, inbox));
}
internal StaThread(IQueueReader <SendOrPostCallbackItem> reader, string threadName)
{
_consumerQueue = reader;
_staThread = new Thread(Run);
_staThread.Name = threadName;
_staThread.SetApartmentState(ApartmentState.STA);
}
public void Shutdown()
{
IQueueReader <T>[] outstandingReaders = null;
lock (ThisLock)
{
if (_queueState == QueueState.Shutdown)
{
return;
}
if (_queueState == QueueState.Closed)
{
return;
}
_queueState = QueueState.Shutdown;
if (_readerQueue.Count > 0 && _itemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader <T> [_readerQueue.Count];
_readerQueue.CopyTo(outstandingReaders, 0);
_readerQueue.Clear();
}
}
if (outstandingReaders != null)
{
for (var i = 0; i < outstandingReaders.Length; i++)
{
outstandingReaders[i].Set(new Item <T>((Exception)null, null));
}
}
}
public void Dispose()
{
bool dispose = false;
lock (ThisLock)
{
if (queueState != QueueState.Closed)
{
queueState = QueueState.Closed;
dispose = true;
}
}
if (dispose)
{
while (readerQueue.Count > 0)
{
IQueueReader reader = readerQueue.Dequeue();
reader.Set(default(Item));
}
while (itemQueue.HasAnyItem)
{
Item item = itemQueue.DequeueAnyItem();
DisposeItem(item);
InvokeDequeuedCallback(item.DequeuedCallback);
}
}
}
// Don't let any more items in. Differs from Close in that we keep around
// existing items in our itemQueue for possible future calls to Dequeue
public void Shutdown(Func <Exception> pendingExceptionGenerator)
{
IQueueReader[] outstandingReaders = null;
lock (ThisLock)
{
if (queueState == QueueState.Shutdown)
{
return;
}
if (queueState == QueueState.Closed)
{
return;
}
this.queueState = QueueState.Shutdown;
if (readerQueue.Count > 0 && this.itemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader[readerQueue.Count];
readerQueue.CopyTo(outstandingReaders, 0);
readerQueue.Clear();
}
}
if (outstandingReaders != null)
{
for (int i = 0; i < outstandingReaders.Length; i++)
{
Exception exception = (pendingExceptionGenerator != null) ? pendingExceptionGenerator() : null;
outstandingReaders[i].Set(new Item(exception, null));
}
}
}
internal StaThread(IQueueReader<SendOrPostCallbackItem> reader, string threadName)
{
_consumerQueue = reader;
_staThread = new Thread(Run);
_staThread.Name = threadName;
_staThread.SetApartmentState(ApartmentState.STA);
}
public override void InitBinding(IServiceProvider serviceProvider, MethodInfo callbackMethod)
{
_serviceProvider = serviceProvider;
_method = callbackMethod;
var metadata = _method.GetCustomAttribute <QueueTriggerAttribute>();
_connection = metadata.Connection;
_queueName = metadata.Queue;
_queueReader = _queueReaderFactory.Create(_connection, _queueName);
_shouldNotify = metadata.Notify;
var parameters = _method.GetParameters();
if (parameters.Length > 2 && parameters.Length < 1)
{
throw new Exception($"Method {_method.Name} must have 1 or 2 parameters");
}
if (parameters.Length == 2 && parameters[1].ParameterType != typeof(DateTimeOffset) && parameters[1].ParameterType != typeof(QueueTriggeringContext))
{
throw new Exception($"Method {_method.Name} second parameter type is {parameters[1].ParameterType.Name}, but should be DateTimeOffset or QueueTriggeringContext");
}
_parameterType = parameters[0].ParameterType;
_hasSecondParameter = parameters.Length == 2;
if (_hasSecondParameter)
{
_useTriggeringContext = parameters[1].ParameterType == typeof(QueueTriggeringContext);
}
_delayStrategy = new RandomizedExponentialStrategy(_minDelay,
metadata.MaxPollingIntervalMs > 0 ? TimeSpan.FromMilliseconds(metadata.MaxPollingIntervalMs) : _maxDelay);
_maxDequeueCount = metadata.MaxDequeueCount;
}
internal StaThread(IQueueReader<SendOrPostCallbackItem> reader)
{
mQueueConsumer = reader;
mStaThread = new Thread(Run);
mStaThread.Name = "STA Worker Thread";
mStaThread.SetApartmentState(ApartmentState.STA);
}
// Used for timeouts. The InputQueue must remove readers from its reader queue to prevent
// dispatching items to timed out readers.
private bool RemoveReader(IQueueReader reader)
{
Fx.Assert(reader != null, "InputQueue.RemoveReader: (reader != null)");
lock (ThisLock)
{
if (_queueState == QueueState.Open || _queueState == QueueState.Shutdown)
{
bool removed = false;
for (int i = _readerQueue.Count; i > 0; i--)
{
IQueueReader temp = _readerQueue.Dequeue();
if (object.ReferenceEquals(temp, reader))
{
removed = true;
}
else
{
_readerQueue.Enqueue(temp);
}
}
return(removed);
}
}
return(false);
}
public bool RemoveReader(IQueueReader <T> reader)
{
lock (ThisLock)
{
if (_queueState == QueueState.Open || _queueState == QueueState.Shutdown)
{
var removed = false;
for (var i = _readerQueue.Count; i > 0; i--)
{
var temp = _readerQueue.Dequeue();
if (ReferenceEquals(temp, reader))
{
removed = true;
}
else
{
_readerQueue.Enqueue(temp);
}
}
return(removed);
}
}
return(false);
}
public void Shutdown()
{
IQueueReader <T>[] outstandingReaders = null;
lock (LockObject)
{
if (_QueueState == QueueState.Shutdown)
{
return;
}
if (_QueueState == QueueState.Closed)
{
return;
}
this._QueueState = QueueState.Shutdown;
if (_QueueReader.Count > 0 && _ItemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader <T> [_QueueReader.Count];
_QueueReader.CopyTo(outstandingReaders, 0);
_QueueReader.Clear();
}
}
if (outstandingReaders != null)
{
for (int i = 0; i < outstandingReaders.Length; i++)
{
outstandingReaders[i].Set(new ExItem <T>((Exception)null, null));
}
}
}
// Used for timeouts. The InputQueue must remove readers from its reader queue to prevent
// dispatching items to timed out readers.
bool RemoveReader(IQueueReader reader)
{
lock (ThisLock)
{
if (queueState == QueueState.Open || queueState == QueueState.Shutdown)
{
bool removed = false;
for (int i = readerQueue.Count; i > 0; i--)
{
IQueueReader temp = readerQueue.Dequeue();
if (object.ReferenceEquals(temp, reader))
{
removed = true;
}
else
{
readerQueue.Enqueue(temp);
}
}
return(removed);
}
}
return(false);
}
static void Main(string[] args)
{
var services = new ServiceCollection();
ConfigureServices(services);
IQueueReader reader = null;
IQueueWriter writer = null;
using (ServiceProvider serviceProvider = services.BuildServiceProvider())
{
writer = serviceProvider.GetService <IQueueWriter>();
reader = serviceProvider.GetService <IQueueReader>();
}
var putMsg = new QueueMessage()
{
CorrelationId = Guid.NewGuid().ToString(),
Data = "This is a message",
QueueName = "DEV.QUEUE.1"
};
writer.Enqueue(putMsg);
var getMsg = reader.Dequque();
Console.WriteLine(getMsg.Data);
}
internal StaThread(IQueueReader<SendOrPostCallbackItem> reader, SynchronizationContext syncContext)
{
mQueueConsumer = reader;
this.syncContext = syncContext;
mStaThread = new Thread(Run);
mStaThread.Name = "STA Worker Thread";
mStaThread.SetApartmentState(ApartmentState.STA);
}
public DeduplicatingQueueReader(IQueueReader queueReader, int?gen0Limit = null, TimeSpan?timeToLive = null)
{
Contract.Requires(queueReader != null);
_queueReader = queueReader;
_hashByLabel = new Cache <string, string>(gen0Limit, timeToLive ?? TimeSpan.FromDays(1));
_hash = SHA1.Create();
_hash.Initialize();
}
public static GZipStrategy CompressorFactory(IQueueReader queueReader, IQueueWriter queueWriter, IGZipProgress gZipProgress)
{
var fileCompressor = new FileCompressor(queueReader, queueWriter);
var fileReader = new FileReaderForCompress(queueReader, gZipProgress);
var fileWriter = new FileWriterForCompress(queueWriter, gZipProgress);
return(new GZipStrategy(fileCompressor, fileReader, fileWriter));
}
internal StaThread(IQueueReader <SendOrPostCallbackItem> reader, SynchronizationContext syncContext)
{
mQueueConsumer = reader;
this.syncContext = syncContext;
mStaThread = new Thread(Run);
mStaThread.Name = "STA Worker Thread";
mStaThread.SetApartmentState(ApartmentState.STA);
}
public SmsQueueConsumer(IQueueReader queueReader, ISmsSender emailSender, ISmsTextGenerator smsTextGenerator, IAlternativeSmsSender alternativeSmsSender)
{
_queueReader = queueReader;
_smsSender = emailSender;
_smsTextGenerator = smsTextGenerator;
_alternativeSmsSender = alternativeSmsSender;
InitQueues();
}
public void Dispatch()
{
IQueueReader reader = null;
Item item = new Item();
IQueueReader[] outstandingReaders = null;
IQueueWaiter[] waiters = null;
bool itemAvailable = true;
lock (ThisLock)
{
itemAvailable = !((_queueState == QueueState.Closed) || (_queueState == QueueState.Shutdown));
this.GetWaiters(out waiters);
if (_queueState != QueueState.Closed)
{
_itemQueue.MakePendingItemAvailable();
if (_readerQueue.Count > 0)
{
item = _itemQueue.DequeueAvailableItem();
reader = _readerQueue.Dequeue();
if (_queueState == QueueState.Shutdown && _readerQueue.Count > 0 && _itemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader[_readerQueue.Count];
_readerQueue.CopyTo(outstandingReaders, 0);
_readerQueue.Clear();
itemAvailable = false;
}
}
}
}
if (outstandingReaders != null)
{
if (s_completeOutstandingReadersCallback == null)
{
s_completeOutstandingReadersCallback = new Action <object>(CompleteOutstandingReadersCallback);
}
ActionItem.Schedule(s_completeOutstandingReadersCallback, outstandingReaders);
}
if (waiters != null)
{
CompleteWaitersLater(itemAvailable, waiters);
}
if (reader != null)
{
InvokeDequeuedCallback(item.DequeuedCallback);
reader.Set(item);
}
}
public void Dispatch()
{
IQueueReader reader = null;
Item item = new Item();
IQueueReader[] outstandingReaders = null;
IQueueWaiter[] waiters = null;
bool itemAvailable = true;
lock (ThisLock)
{
itemAvailable = !((queueState == QueueState.Closed) || (queueState == QueueState.Shutdown));
this.GetWaiters(out waiters);
if (queueState != QueueState.Closed)
{
itemQueue.MakePendingItemAvailable();
if (readerQueue.Count > 0)
{
item = itemQueue.DequeueAvailableItem();
reader = readerQueue.Dequeue();
if (queueState == QueueState.Shutdown && readerQueue.Count > 0 && itemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader[readerQueue.Count];
readerQueue.CopyTo(outstandingReaders, 0);
readerQueue.Clear();
itemAvailable = false;
}
}
}
}
if (outstandingReaders != null)
{
if (completeOutstandingReadersCallback == null)
{
completeOutstandingReadersCallback = new WaitCallback(CompleteOutstandingReadersCallback);
}
ThreadPool.QueueUserWorkItem(completeOutstandingReadersCallback, outstandingReaders);
}
if (waiters != null)
{
CompleteWaitersLater(itemAvailable, waiters);
}
if (reader != null)
{
InvokeDequeuedCallback(item.DequeuedCallback);
reader.Set(item);
}
}
/// <summary>
/// Azures the index builder.
/// </summary>
/// <param name="repository">The repository.</param>
/// <param name="searchProvider">The search provider.</param>
/// <param name="messageSender">The message sender.</param>
/// <param name="consumerFactory">The consumer factory.</param>
/// <param name="observer">The observer.</param>
/// <param name="queueReader">The queue reader.</param>
/// <param name="indexBuilders">The index builders.</param>
public SearchIndexController(IBuildSettingsRepository repository, ISearchProvider searchProvider, IMessageSender messageSender, IConsumerFactory consumerFactory, ISystemObserver observer, IQueueReader queueReader, ISearchIndexBuilder[] indexBuilders)
{
_repository = repository;
_messageSender = messageSender;
_observer = observer;
_queueReader = queueReader;
_consumerFactory = consumerFactory;
_searchProvider = searchProvider;
_indexBuilders = indexBuilders;
}
private async Task MoveToPoisonQueue(IQueueMessage message)
{
if (_poisonQueueReader == null)
{
_poisonQueueReader = _queueReaderFactory.Create(_queueName + PoisonSuffix);
}
await _poisonQueueReader.AddMessageAsync(message.AsString);
await _queueReader.FinishMessageAsync(message);
}
/// <summary>
/// Azures the index builder.
/// </summary>
/// <param name="repository">The repository.</param>
/// <param name="searchProvider">The search provider.</param>
/// <param name="messageSender">The message sender.</param>
/// <param name="consumerFactory">The consumer factory.</param>
/// <param name="observer">The observer.</param>
/// <param name="queueReader">The queue reader.</param>
/// <param name="indexBuilders">The index builders.</param>
public SearchIndexController(IBuildSettingsRepository repository, ISearchProvider searchProvider, IMessageSender messageSender, IConsumerFactory consumerFactory, ISystemObserver observer, IQueueReader queueReader, ISearchIndexBuilder[] indexBuilders)
{
_repository = repository;
_messageSender = messageSender;
_observer = observer;
_queueReader = queueReader;
_consumerFactory = consumerFactory;
_searchProvider = searchProvider;
_indexBuilders = indexBuilders;
}
public void Dispatch()
{
IQueueReader <T> reader = null;
ExItem <T> item = new ExItem <T>();
IQueueReader <T>[] outstandingReaders = null;
IQueueWaiter[] waiters = null;
bool itemAvailable = true;
lock (LockObject)
{
itemAvailable = (_QueueState != QueueState.Closed && _QueueState != QueueState.Shutdown);
this.getWaiters(out waiters);
if (_QueueState != QueueState.Closed)
{
_ItemQueue.MakePendingItemAvailable();
if (_QueueReader.Count > 0)
{
item = _ItemQueue.DequeueAvailableItem();
reader = _QueueReader.Dequeue();
if (_QueueState == QueueState.Shutdown &&
_QueueReader.Count > 0 &&
_ItemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader <T> [_QueueReader.Count];
_QueueReader.CopyTo(outstandingReaders, 0);
_QueueReader.Clear();
itemAvailable = false;
}
}
}
}
if (outstandingReaders != null)
{
if (_CompleteOutstandingReadersCallback == null)
{
_CompleteOutstandingReadersCallback = completeOutstandingReadersCallback;
}
ThreadPool.QueueUserWorkItem(_CompleteOutstandingReadersCallback, outstandingReaders);
}
if (waiters != null)
{
completeWaitersLater(itemAvailable, waiters);
}
if (reader != null)
{
invokeDequeueCallback(item.Callback);
reader.Set(item);
}
}
public void Dispatch()
{
IQueueReader <T> reader = null;
var item = new Item <T>();
IQueueReader <T>[] outstandingReaders = null;
IQueueWaiter[] waiters = null;
var itemAvailable = true;
lock (ThisLock)
{
itemAvailable = !((_queueState == QueueState.Closed) || (_queueState == QueueState.Shutdown));
GetWaiters(out waiters);
if (_queueState != QueueState.Closed)
{
_itemQueue.MakePendingItemAvailable();
if (_readerQueue.Count > 0)
{
item = _itemQueue.DequeueAvailableItem();
reader = _readerQueue.Dequeue();
if (_queueState == QueueState.Shutdown && _readerQueue.Count > 0 && _itemQueue.ItemCount == 0)
{
outstandingReaders = new IQueueReader <T> [_readerQueue.Count];
_readerQueue.CopyTo(outstandingReaders, 0);
_readerQueue.Clear();
itemAvailable = false;
}
}
}
}
if (outstandingReaders != null)
{
ThreadPool.QueueUserWorkItem(CompleteOutstandingReadersCallback, outstandingReaders);
}
if (waiters != null)
{
CompleteWaitersLater(itemAvailable, waiters);
}
if (reader != null)
{
InvokeDequeuedCallback(item.DequeuedCallback);
reader.Set(item);
}
}
public void Dispatch()
{
IQueueReader <T> reader = null;
Item <T> item = new Item <T>();
IQueueReader <T>[] array = null;
IQueueWaiter <T>[] waiters = null;
bool itemAvailable = true;
lock (this.ThisLock)
{
itemAvailable = (this.queueState != QueueState <T> .Closed) && (this.queueState != QueueState <T> .Shutdown);
this.GetWaiters(out waiters);
if (this.queueState != QueueState <T> .Closed)
{
this.itemQueue.MakePendingItemAvailable();
if (this.readerQueue.Count > 0)
{
item = this.itemQueue.DequeueAvailableItem();
reader = this.readerQueue.Dequeue();
if (((this.queueState == QueueState <T> .Shutdown) && (this.readerQueue.Count > 0)) && (this.itemQueue.ItemCount == 0))
{
array = new IQueueReader <T> [this.readerQueue.Count];
this.readerQueue.CopyTo(array, 0);
this.readerQueue.Clear();
itemAvailable = false;
}
}
}
}
if (array != null)
{
if (InputQueue <T> .completeOutstandingReadersCallback == null)
{
InputQueue <T> .completeOutstandingReadersCallback = new Action <object>(InputQueue <T> .CompleteOutstandingReadersCallback);
}
ActionItem.Schedule(InputQueue <T> .completeOutstandingReadersCallback, array);
}
if (waiters != null)
{
InputQueue <T> .CompleteWaitersLater(itemAvailable, waiters);
}
if (reader != null)
{
InputQueue <T> .InvokeDequeuedCallback(item.DequeuedCallback);
reader.Set(item);
}
}
public QueueWork(
ILog <QueueWork> log,
IPerfCounters counters,
IFindQueueHandlers findHandlers,
IQueueReader queueReader,
IBusDataAccess dataAccess,
ISagaMessageMapManager sagaMessageMapManager)
{
_log = log;
_counters = counters;
_findHandlers = findHandlers;
_queueReader = queueReader;
_dataAccess = dataAccess;
_sagaMessageMapManager = sagaMessageMapManager;
}
private async Task MoveToPoisonQueue(IQueueMessage message, string newMessageVersion)
{
newMessageVersion = newMessageVersion ?? message.AsString;
if (_poisonQueueReader == null)
{
_poisonQueueReader = _queueReaderFactory.Create(_connection, _queueName + PoisonSuffix);
}
await _poisonQueueReader.AddMessageAsync(newMessageVersion);
await _queueReader.FinishMessageAsync(message);
if (_shouldNotify)
{
await _notifier.NotifyAsync($"Msg put to {_queueName + PoisonSuffix}, data: {newMessageVersion}");
}
}
public void SetUp()
{
_currentDateUtc = DateTime.UtcNow;
_queue = new Queue
{
Body = QueueMessage,
IsProcessed = false,
Processed = null
};
_timeService = new Mock <ITimeService>();
_timeService.Setup(_ => _.GetUtc()).Returns(_currentDateUtc);
_queueRepository = new Mock <IQueueRepository>();
_target = new DatabaseQueueReader(_timeService.Object, _queueRepository.Object, Key);
}
public DispatcherProcess(Action<byte[]> dispatcher, IQueueReader inbox)
{
_dispatcher = dispatcher;
_inbox = inbox;
}
public SingleThreadConsumingProcess(ISystemObserver observer, IConsumerFactory consumerFactory, IQueueReader queueReader)
{
_consumerFactory = consumerFactory;
_observer = observer;
_queueReader = queueReader;
}
public void Handle(IQueueReader inbox, Action<ImmutableEnvelope> lambda, string name = null)
{
var dispatcherName = name ?? "inbox-" + Interlocked.Increment(ref _counter);
var dispatcher = new EnvelopeDispatcher(lambda, Streamer, Quarantine, Duplication, dispatcherName);
AddTask(new DispatcherProcess(dispatcher.Dispatch, inbox));
}
public void Dispatch(IQueueReader inbox, Action<byte[]> lambda)
{
Processes.Add(new DispatcherProcess(lambda, inbox));
}
