public void WriteMultipleData_OneFramePerWrite()
{
using (Stream transport = new QueueStream())
using (WriteQueue queue = new WriteQueue(transport))
{
Task pumpTask = queue.PumpToStreamAsync();
Stream output = new OutputStream(1, Framing.Priority.Pri1, queue);
FrameReader reader = new FrameReader(transport, false, CancellationToken.None);
int dataLength = 100;
output.Write(new byte[dataLength], 0, dataLength);
output.Write(new byte[dataLength * 2], 0, dataLength * 2);
output.Write(new byte[dataLength * 3], 0, dataLength * 3);
Frame frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(dataLength, frame.FrameLength);
frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(dataLength * 2, frame.FrameLength);
frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(dataLength * 3, frame.FrameLength);
}
}
public void WriteMoreDataThanCredited_OnlyCreditedDataWritten()
{
using (Stream transport = new QueueStream())
using (WriteQueue queue = new WriteQueue(transport))
{
Task pumpTask = queue.PumpToStreamAsync();
OutputStream output = new OutputStream(1, Framing.Priority.Pri1, queue);
FrameReader reader = new FrameReader(transport, false, CancellationToken.None);
int dataLength = 0x1FFFF; // More than the 0x10000 default
Task writeTask = output.WriteAsync(new byte[dataLength], 0, dataLength);
Assert.False(writeTask.IsCompleted);
Frame frame = reader.ReadFrameAsync().Result;
Assert.False(frame.IsControl);
Assert.Equal(0x10000, frame.FrameLength);
Task<Frame> nextFrameTask = reader.ReadFrameAsync();
Assert.False(nextFrameTask.IsCompleted);
// Free up some space
output.AddFlowControlCredit(10);
frame = nextFrameTask.Result;
Assert.False(frame.IsControl);
Assert.Equal(10, frame.FrameLength);
nextFrameTask = reader.ReadFrameAsync();
Assert.False(nextFrameTask.IsCompleted);
}
}
public void Test_QueueStream()
{
using (FileStream stream1 = new FileStream("QueueStream1.tmp", FileMode.Create))
using (QueueStream queueStream = new QueueStream(stream1, StreamMode.Write, 1024 * 1024 * 4, _bufferManager))
using (FileStream stream2 = new FileStream("QueueStream2.tmp", FileMode.Create))
{
byte[] buffer = new byte[1024];
for (int i = 0; i < 1024 * 32; i++)
{
_random.NextBytes(buffer);
queueStream.Write(buffer, 0, buffer.Length);
stream2.Write(buffer, 0, buffer.Length);
}
}
using (FileStream stream1 = new FileStream("QueueStream1.tmp", FileMode.Open))
using (FileStream stream2 = new FileStream("QueueStream2.tmp", FileMode.Open))
{
Assert.IsTrue(CollectionUtilities.Equals(Sha256.ComputeHash(stream1), Sha256.ComputeHash(stream2)), "QueueStream #1");
}
using (FileStream stream1 = new FileStream("QueueStream1.tmp", FileMode.Open))
using (QueueStream queueStream = new QueueStream(stream1, StreamMode.Read, 1024 * 1024 * 4, _bufferManager))
using (FileStream stream2 = new FileStream("QueueStream2.tmp", FileMode.Open))
{
Assert.IsTrue(CollectionUtilities.Equals(Sha256.ComputeHash(queueStream), Sha256.ComputeHash(stream2)), "QueueStream #2");
}
}
public async Task TestIisHostReceiver()
{
touched = 0;
CancellationTokenSource tokenSource = new CancellationTokenSource();
// Start the listener on another thread.
Task t = await Task.Factory.StartNew(Listener2, tokenSource.Token);
var stopwatch = new Stopwatch();
Assert.Equal(0, Interlocked.Read(ref touched));
var messages = new List <Message>();
var tags = new Dictionary <string, string>
{
{ "key", "value" }
};
for (int i = 0; i < 1000; i++)
{
messages.Add(
new Message("Order.Test.OutQ.Listener", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()), string.Empty, Guid.NewGuid().ToString(), tags)
);
}
Assert.Equal(0, Interlocked.Read(ref touched));
QueueStream queue = new QueueStream("localhost:50000", "Demo.Test.IISHost", null);
stopwatch.Start();
await queue.Send(new SendRequest(messages));
stopwatch.Stop();
Logger.Technical().Debug($"Messages sent in {stopwatch.ElapsedMilliseconds} ms.").Log();
stopwatch.Reset();
stopwatch.Start();
do
{
await Task.Delay(10);
} while (Interlocked.Read(ref touched) < 1000 && stopwatch.ElapsedMilliseconds < 10000);
stopwatch.Stop();
Logger.Technical().Debug($"Messages read in {stopwatch.ElapsedMilliseconds} ms.").Log();
tokenSource.Cancel();
await Task.Delay(150);
Assert.Equal(1000, Interlocked.Read(ref touched));
Task.WaitAll(t);
queue.Close();
}
public override long Seek(long offset, SeekOrigin origin)
{
try
{
return(Stream.Seek(offset, origin));
}
catch (ObjectDisposedException ex)
{
Stream = new QueueStream();
Cancel();
StartDownload();
return(Stream.Seek(offset, origin));
}
}
private async Task Listener2(object parameter)
{
CancellationToken cancellationToken = (CancellationToken)parameter;
QueueStream queue = new QueueStream("localhost:50000", "Demo.Test.Listener", null);
PollRequest pollRequest = new()
{
Queue = "Order.Test.OutQ.Listener",
WaitTimeout = 1000,
MaxItems = 100,
AutoAck = false
};
var messagesToAck = new ConcurrentBag <Message>();
do
{
PollResponse response = await queue.Poll(pollRequest);
if (!response.HasMessages)
{
continue;
}
Logger.Technical().Debug($"{response.Messages.Count()} message(s) received from the PollResponse.").Log();
Parallel.ForEach(response.Messages, async msg =>
{
try
{
Assert.Equal(1, msg.Tags.Count());
Assert.True(Guid.TryParse(Encoding.UTF8.GetString(msg.Body), out var guid));
await Task.Delay(100);
msg.Ack();
Interlocked.Increment(ref touched);
}
catch (Exception ex)
{
Logger.Technical().Exception(ex).Log();
msg.NAck();
}
});
} while (!cancellationToken.IsCancellationRequested);
queue.Close();
}
}
/// <summary>
/// All the messages must be of the same type.
/// </summary>
/// <param name="messages"></param>
private void SendToQueueInBatch(IEnumerable <QueueMessage> messages)
{
using (var activity = _activitySource?.StartActivity("Send messages to KubeMQ", ActivityKind.Producer))
{
var definition = GetDefiniton(messages.First().Value.GetType());
var policy = definition.RetryPolicy(Policy.Handle <Exception>());
QueueStream queue = null;
policy.Execute(() => queue = _queueStreamManager.Get(definition));
if (null == queue)
{
throw new NullReferenceException($"No queue was created for Address: {definition.Address} and Identitfier {definition.Identifier}");
}
var serializer = GetSerializer(definition);
var _messages = new List <Message>(messages.Count());
string activityId = _applicationContext?.Principal?.ActivityID.ToString() ?? Guid.NewGuid().ToString();
using (var serializerActivity = _activitySource?.StartActivity("Serialize.", ActivityKind.Producer))
{
serializerActivity?.AddTag("Count", messages.Count());
// I use messageType because we consider that we send a message of the same type...
// Doing it by ChannelParameter will be more efficient...
foreach (var message in messages)
{
_messages.Add(
new Message(definition.Namespace, serializer.Serialize(message.Value), String.Empty, Guid.NewGuid().ToString(), BuildTags(message.Value.GetType(), message.Tags, activityId))
{
Policy = BuildPolicy(definition, message.SendAfterSeconds, message.ExpireAfterSeconds)
}
);
}
}
policy.Execute(() =>
{
using (var sendActivity = _activitySource?.StartActivity("Send to KubeMQ", ActivityKind.Producer))
queue.Send(new SendRequest(_messages));
});
}
}
public void Send <TMessage>(TMessage message, Dictionary <string, string> tags = null, uint?sendAfterSeconds = null, uint?expireAfterSeconds = null) where TMessage : notnull
{
using (var activity = _activitySource?.StartActivity("Send message to KubeMQ", ActivityKind.Producer))
{
activity?.AddTag("Delay message [s]", sendAfterSeconds);
activity?.AddTag("Expire message after [s]", expireAfterSeconds);
if (message is IEnumerable)
{
throw new ArgumentException("Collection are not accepted!");
}
var messageType = typeof(TMessage);
var definition = GetDefiniton(messageType);
var policy = definition.RetryPolicy(Policy.Handle <Exception>());
QueueStream queue = null;
policy.Execute(() => queue = _queueStreamManager.Get(definition));
if (null == queue)
{
throw new NullReferenceException($"No queue was created for Address: {definition.Address} and Identitfier {definition.Identifier}");
}
var serializer = GetSerializer(definition);
string activityId = _applicationContext?.Principal?.ActivityID.ToString() ?? Guid.NewGuid().ToString();
Message _message = null;
using (var serializerActivity = _activitySource?.StartActivity("Serialize.", ActivityKind.Producer))
_message = new Message(definition.Namespace, serializer.Serialize(message), String.Empty, Guid.NewGuid().ToString(), BuildTags(messageType, tags, activityId))
{
Policy = BuildPolicy(definition, sendAfterSeconds, expireAfterSeconds)
};
using (var sendActivity = _activitySource?.StartActivity("Send to KubeMQ", ActivityKind.Producer))
queue.Send(new SendRequest(new List <Message>(1)
{
_message
}));
}
}
/// <summary>
/// 初始化参数信息。
/// </summary>
private void Init(int bufferCapacity)
{
//初始化接收发送参数
this.receiveArgs = new SocketAsyncEventArgs();
this.receiveArgs.Completed += this.ProcessReceive;
this.sendArgs = new SocketAsyncEventArgs();
//初始化接收发送队列
this.receiveQueue = new ConcurrentQueue <Buffer <byte> >();
this.sendQueue = new ConcurrentQueue <byte[]>();
//Buffer管理器
this.bufferManager = new BufferManager(bufferCapacity);
//解析数据的队列
this.stream = new QueueStream();
this.receiveFrame = new TransferFrame(digestor);
this.sendFrame = new TransferFrame(digestor);
}
static void Main()
{
var dump = File.Create("/tmp/queued-result", 8192);
var qs = new QueueStream("/tmp/storage");
byte [] buffer = new byte [8192];
int count;
var t = new Thread((o) => {
var request = (HttpWebRequest)WebRequest.Create(url);
var response = request.GetResponse();
var webStream = response.GetResponseStream();
var webBuffer = new byte [8192];
int c;
while ((c = webStream.Read(webBuffer, 0, webBuffer.Length)) != 0)
{
Console.Write(".");
var start = DateTime.UtcNow;
qs.Push(webBuffer, 0, c);
}
Console.WriteLine("Done reading from the web");
qs.Done();
});
t.Start();
long total = 0;
while ((count = qs.Read(buffer, 0, buffer.Length)) != 0)
{
total += count;
Console.WriteLine("Slowly reading {0}, total={1}", count, total);
dump.Write(buffer, 0, count);
Thread.Sleep(1000);
}
}
static void Main ()
{
var dump = File.Create ("/tmp/queued-result", 8192);
var qs = new QueueStream ("/tmp/storage");
byte [] buffer = new byte [8192];
int count;
var t = new Thread ((o) => {
var request = (HttpWebRequest) WebRequest.Create (url);
var response = request.GetResponse ();
var webStream = response.GetResponseStream ();
var webBuffer = new byte [8192];
int c;
while ((c = webStream.Read (webBuffer, 0, webBuffer.Length)) != 0){
Console.Write (".");
var start = DateTime.UtcNow;
qs.Push (webBuffer, 0, c);
}
Console.WriteLine ("Done reading from the web");
qs.Done ();
});
t.Start ();
long total = 0;
while ((count = qs.Read (buffer, 0, buffer.Length)) != 0){
total += count;
Console.WriteLine ("Slowly reading {0}, total={1}", count, total);
dump.Write (buffer, 0, count);
Thread.Sleep (1000);
}
}
//RAM stream -> File stream
private void _monitor_thread_callback()
{
//设置线程启动相关变量
_monitor_thread_created.Set();
_start_time = DateTime.Now;
//管理任务状态
bool pause_to_start = false;
lock (_thread_flag_lock)
{
pause_to_start = ((_download_thread_flag & 0xffffff) & (_DOWNLOAD_THREAD_FLAG_PAUSED | _DOWNLOAD_THREAD_FLAG_PAUSE_REQUESTED)) != 0;
_download_thread_flag = (_download_thread_flag | _DOWNLOAD_THREAD_FLAG_STARTED) & ~_DOWNLOAD_THREAD_FLAG_START_REQUESTED;
if (pause_to_start)
{
_download_thread_flag = (_download_thread_flag & ~_DOWNLOAD_THREAD_FLAG_PAUSED);
}
}
//缓存最大线程数，避免中途更改
var max_thread = _max_thread;
//预分配空间（todo：改为set-length，提高效率）
_downloaded_size = 0;
try { PreAllocBlockStarted?.Invoke(this, new EventArgs()); } catch { }
if (_file_stream.Length != (long)_data.Size)
{
if (_file_stream.Length > (long)_data.Size)
{
_file_stream.SetLength((long)_data.Size); //文件长度大于预分配长度，直接截断
}
else
{
//从当前长度开始按0填充
_file_stream.Seek(0, SeekOrigin.End);
_downloaded_size = _file_stream.Length;
var blank_buffer = new byte[_MIN_IO_FLUSH_DATA_LENGTH];
long write_length = 0;
do
{
write_length = (long)_data.Size - _file_stream.Length;
int len = (int)Math.Min(_MIN_IO_FLUSH_DATA_LENGTH, write_length);
_file_stream.Write(blank_buffer, 0, len);
_downloaded_size += len;
} while (write_length > 0 && ((_download_thread_flag & 0xffffff) & (_DOWNLOAD_THREAD_FLAG_PAUSE_REQUESTED | _DOWNLOAD_THREAD_FLAG_STOP_REQUESTED)) == 0);
}
_file_stream.Seek(0, SeekOrigin.Begin);
}
try { PreAllocBlockFinished?.Invoke(this, new EventArgs()); } catch { }
_downloaded_size = 0;
//分配多线程的变量数组
_guid_list = new Guid[max_thread];
_last_receive = new DateTime[max_thread];
_buffer_stream = new QueueStream[max_thread];
_thread_data_lock = new object[max_thread];
_request = new NetStream[max_thread];
_position = new ulong[max_thread];
_io_position = new long[max_thread];
for (int i = 0; i < max_thread; i++)
{
_buffer_stream[i] = new QueueStream();
_thread_data_lock[i] = new object();
_request[i] = new NetStream();
_request[i].CookieKey = _cookie_key;
}
var buffer = new byte[_MIN_IO_FLUSH_DATA_LENGTH];
var next_loop_time = DateTime.Now.AddSeconds(1);
//监控循环
#region monitor loop
while (true)
{
//当url的有效时间已过时，刷新url
if (_url_expire_time < DateTime.Now)
{
_url_expire_time = _url_expire_time.AddSeconds(30);
_url_fail_to_fetch_count = 0;
_api.GetAccount(_data.AccountID).GetLocateDownloadLinkAsync(_data.Path, _main_url_refresh_callback);
}
//管理任务标识
#region thread flag handling
bool ispause = false, iscancel = false;
if (((_download_thread_flag & 0xffffff) & _DOWNLOAD_THREAD_FLAG_PAUSE_REQUESTED) != 0)
{
ispause = true;
}
if (((_download_thread_flag & 0xffffff) & _DOWNLOAD_THREAD_FLAG_STOP_REQUESTED) != 0)
{
iscancel = true;
}
if (ispause)
{
//pause event
for (int i = 0; i < _request.Length; i++)
{
lock (_thread_data_lock[i])
{
_request[i].Close();
if (_guid_list[i] != Guid.Empty)
{
_dispatcher.ReleaseTask(_guid_list[i]);
_guid_list[i] = Guid.Empty;
}
//flushing buffer stream
if ((_guid_list[i] == Guid.Empty && _buffer_stream[i].Length > 0) || (_buffer_stream[i].Length > _MIN_IO_FLUSH_DATA_LENGTH))
{
_file_stream.Seek(_io_position[i], SeekOrigin.Begin);
while ((_guid_list[i] == Guid.Empty && _buffer_stream[i].Length > 0) || (_buffer_stream[i].Length > _MIN_IO_FLUSH_DATA_LENGTH))
{
int rc = _buffer_stream[i].Read(buffer, 0, _MIN_IO_FLUSH_DATA_LENGTH);
_file_stream.Write(buffer, 0, rc);
_io_position[i] += rc;
}
}
}
_file_stream.Flush();
}
_request = null;
_urls = null;
_guid_list = null;
_last_receive = null;
_position = null;
_thread_data_lock = null;
_monitor_thread = null;
_end_time = DateTime.Now;
_download_thread_flag = (_download_thread_flag | _DOWNLOAD_THREAD_FLAG_PAUSED) & ~(_DOWNLOAD_THREAD_FLAG_PAUSE_REQUESTED | _DOWNLOAD_THREAD_FLAG_STARTED);
return;
}
if (iscancel)
{
//cancel event
_file_stream.Close();
for (int i = 0; i < _request.Length; i++)
{
lock (_thread_data_lock[i])
{
_request[i].Close();
if (_guid_list[i] != Guid.Empty)
{
_dispatcher.ReleaseTask(_guid_list[i]);
_guid_list[i] = Guid.Empty;
}
}
}
_request = null;
_urls = null;
_guid_list = null;
_last_receive = null;
_position = null;
_thread_data_lock = null;
_file_stream.Dispose();
_file_stream = null;
_monitor_thread = null;
_end_time = DateTime.Now;
_download_thread_flag = (_download_thread_flag | _DOWNLOAD_THREAD_FLAG_STOPPED) & ~(_DOWNLOAD_THREAD_FLAG_STOP_REQUESTED | _DOWNLOAD_THREAD_FLAG_STARTED);
return;
}
#endregion
//url合理检测
if (_urls == null || _urls.Length == 0)
{
Thread.Sleep(100);
continue;
}
int started_tasks = 0;
for (int i = 0; i < _request.Length; i++)
{
lock (_thread_data_lock[i])
{
//将内存的缓存数据流写入到硬盘中
if ((_guid_list[i] == Guid.Empty && _buffer_stream[i].Length > 0) || (_buffer_stream[i].Length > _MIN_IO_FLUSH_DATA_LENGTH))
{
_file_stream.Seek(_io_position[i], SeekOrigin.Begin);
while ((_guid_list[i] == Guid.Empty && _buffer_stream[i].Length > 0) || (_buffer_stream[i].Length > _MIN_IO_FLUSH_DATA_LENGTH))
{
int rc = _buffer_stream[i].Read(buffer, 0, _MIN_IO_FLUSH_DATA_LENGTH);
_file_stream.Write(buffer, 0, rc);
_io_position[i] += rc;
}
}
//检测任务分配状态，自动开始新的分段下载
if (_guid_list[i] == Guid.Empty && (DateTime.Now - _last_receive[i]).TotalSeconds > 3.0)
{
if (started_tasks < _PARALLEL_START_REQUEST_COUNT)
{
_guid_list[i] = _dispatcher.AllocateNewTask(out _position[i]);
if (_guid_list[i] != Guid.Empty)
{
_last_receive[i] = DateTime.Now;
try
{
_request[i].UserAgent = "netdisk;5.7.2.3;PC;PC-Windows;10.0.16299;WindowsBaiduYunGuanJia";
_request[i].HttpGetAsync(_urls[i % _urls.Length], _data_transfer_callback, new _temp_strcut {
id = _guid_list[i], index = i
}, range: (long)_position[i]);
_io_position[i] = (long)_position[i];
}
catch { }
started_tasks++;
}
}
}
//自动中断超时的请求
else if (_guid_list[i] != Guid.Empty && (DateTime.Now - _last_receive[i]).TotalSeconds > 35.0)
{
_request[i].Close();
_request[i] = new NetStream();
_dispatcher.ReleaseTask(_guid_list[i]);
_guid_list[i] = Guid.Empty;
}
}
}
_current_bytes = 0;
//更新速度
var cur_len = _dispatcher.CompletedLength;
_last_5s_length.RemoveFirst();
_last_5s_length.AddLast(cur_len);
_average_speed_total = cur_len / (DateTime.Now - _start_time).TotalSeconds;
_average_speed_5s = (_last_5s_length.Last.Value - _last_5s_length.First.Value) / 5.0;
_downloaded_size = (long)cur_len;
Tracer.GlobalTracer.TraceInfo("Downloading: " + cur_len + "/" + _data.Size + " [" + (_average_speed_5s / 1024).ToString("0.00") + "KB/s]");
if (cur_len == _data.Size)
{
break;
}
//时钟控制（1s）
var ts = (next_loop_time - DateTime.Now).TotalMilliseconds;
next_loop_time = next_loop_time.AddSeconds(1);
if (ts >= 1.0)
{
Thread.Sleep((int)ts);
}
}
#endregion
//下载完成
_file_stream.Close();
for (int i = 0; i < _request.Length; i++)
{
lock (_thread_data_lock[i])
{
_request[i].Close();
}
}
_request = null;
_urls = null;
_guid_list = null;
_last_receive = null;
_position = null;
_thread_data_lock = null;
_file_stream.Dispose();
_file_stream = null;
_monitor_thread = null;
_end_time = DateTime.Now;
//文件解密
if (_data.Path.EndsWith(".bcsd"))
{
_download_thread_flag |= _DOWNLOAD_THREAD_FLAG_DECRYPTING;
try { DecryptStarted?.Invoke(this, new EventArgs()); } catch { }
_decrypt_file();
_download_thread_flag = _download_thread_flag & ~_DOWNLOAD_THREAD_FLAG_DECRYPTING;
try { DecryptFinished?.Invoke(this, new EventArgs()); } catch { }
}
_download_thread_flag = (_download_thread_flag | _DOWNLOAD_THREAD_FLAG_FINISHED) & ~_DOWNLOAD_THREAD_FLAG_STARTED;
try { TaskFinished?.Invoke(this, new EventArgs()); }
catch { }
}
public void Twister()
{
var queue = new QueueStream();
TestUtils.RunOneWayTwister(queue, queue);
}
public async Task TestBasicSendAndReceive()
{
QueueStream queue = new QueueStream("localhost:50000", "Demo.Test.Basic", null);
var messages = new List <Message>();
var tags = new Dictionary <string, string>
{
{ "key", "value" }
};
for (int i = 0; i < 1000; i++)
{
messages.Add(
new Message("Order.Test.OutQ.Basic", Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()), string.Empty, Guid.NewGuid().ToString(), tags)
);
}
var res = await queue.Send(new SendRequest(messages));
PollRequest pollRequest = new()
{
Queue = "Order.Test.OutQ.Basic",
WaitTimeout = 1000,
MaxItems = 100,
AutoAck = false
};
var read = 0;
while (read < 1000)
{
PollResponse response = await queue.Poll(pollRequest);
if (!response.HasMessages)
{
continue;
}
Logger.Technical().Debug($"{response.Messages.Count()} message(s) received from the PollResponse.").Log();
Parallel.ForEach(response.Messages, msg =>
{
try
{
Assert.Equal(1, msg.Tags.Count());
Assert.True(Guid.TryParse(Encoding.UTF8.GetString(msg.Body), out var guid));
msg.Ack();
}
catch (Exception)
{
msg.NAck();
}
});
read += response.Messages.Count();
}
Assert.Equal(1000, read);
queue.Close();
}
private void SendToQueueInBatch(IEnumerable <QueueMessage> messages, int splitByQueues)
{
using (var activity = _activitySource?.StartActivity("Send messages to KubeMQ", ActivityKind.Producer))
{
activity?.AddTag("Split", splitByQueues);
if (splitByQueues < 2)
{
throw new ArgumentOutOfRangeException("Number of queues must be greater than 1!");
}
if (splitByQueues > 10)
{
splitByQueues = 10;
_logger.Technical().Warning("Cap the number of queues to 10").Log();
}
var messageType = messages.First().Value.GetType();
// All the messages are assumed to be of the same type.
var definition = GetDefiniton(messageType);
var policy = definition.RetryPolicy(Policy.Handle <Exception>());
QueueStream queue = null;
policy.Execute(() => queue = _queueStreamManager.Get(definition));
if (null == queue)
{
throw new NullReferenceException($"No queue was created for Address: {definition.Address} and Identitfier {definition.Identifier}");
}
var serializer = GetSerializer(definition);
string activityId = _applicationContext?.Principal?.ActivityID.ToString() ?? Guid.NewGuid().ToString();
var messagesByQueue = Math.DivRem(messages.Count(), splitByQueues, out var rem);
var _messages = new List <Message>(messagesByQueue + rem);
int idx = 0;
int queueIdx = 1;
foreach (var message in messages)
{
using (var createactivity = _activitySource?.StartActivity("Create KubeMQ messages", ActivityKind.Producer))
{
createactivity?.AddTag("Count", messages.Count());
createactivity?.AddTag("QueueName", $"{definition.Namespace}.{queueIdx}");
_messages.Add(
new Message($"{definition.Namespace}.{queueIdx}", serializer.Serialize(message.Value), String.Empty, Guid.NewGuid().ToString(), BuildTags(messageType, message.Tags, activityId))
{
Policy = BuildPolicy(definition, message.SendAfterSeconds, message.ExpireAfterSeconds)
}
);
}
idx++;
if (0 == idx % messagesByQueue) // we have on batch.
{
policy.Execute(() =>
{
using (var sendActivity = _activitySource?.StartActivity("Send to KubeMQ", ActivityKind.Producer))
queue.Send(new SendRequest(_messages));
});
queueIdx++;
idx = 0;
_messages.Clear();
}
}
}
}
