public async Task ProduceFromMultipleThreads()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
var t1 = Task.Run(() =>
{
for (int i = 0; i < 500; i++)
{
collection.Add(i * 2);
}
});
var t2 = Task.Run(() =>
{
for (int i = 0; i < 500; i++)
{
collection.Add(i * 2 + 1);
}
});
Task.WhenAll(t1, t2).ContinueWith(t => collection.CompleteAdding());
int counter = 0;
await foreach (var item in collection)
{
counter++;
}
Assert.AreEqual(counter, 1000);
}
public async Task TakeThenComplete()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var MyTask = MyCollection.Take();
Assert.IsFalse(MyTask.IsCompleted, "Took too early");
_ = MyCollection.CompleteAdding();
//****************************************
try
{
_ = await MyTask;
Assert.Fail("Should not reach here");
}
catch (InvalidOperationException)
{
}
Assert.IsTrue(MyCollection.IsAddingCompleted, "Adding not completed");
Assert.IsTrue(MyCollection.IsCompleted, "Collection not completed");
}
public void ConsumeSyncFromMultipleThreads()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
int t1Counter = 0;
int t2Counter = 0;
var t1 = Task.Run(() =>
{
foreach (var item in collection.GetConsumingEnumerable())
{
t1Counter++;
}
});
var t2 = Task.Run(() =>
{
foreach (var item in collection.GetConsumingEnumerable())
{
t2Counter++;
}
});
for (int i = 0; i < 1000; i++)
{
collection.Add(i);
}
collection.CompleteAdding();
t1.Wait();
t2.Wait();
Assert.IsTrue(t1Counter > 400);
Assert.IsTrue(t2Counter > 400);
}
async Task Test()
{
var _asyncStack = new AsyncCollection <int>(
new ConcurrentStack <int>(), maxCount: 1);
// This Add completes immediately.
await _asyncStack.AddAsync(7);
// This Add (asynchronously) waits for the 7 to be removed
// before it enqueues the 13.
await _asyncStack.AddAsync(13);
_asyncStack.CompleteAdding();
while (true)
{
int item;
try
{
item = await _asyncStack.TakeAsync();
}
catch (InvalidOperationException)
{
break;
}
Trace.WriteLine(item);
}
}
private KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout, KafkaOptions kafkaOptions)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_processNetworkstreamTasksAction = ProcessNetworkstreamTasks;
_allowSelfSignedServerCert = kafkaOptions?.TslAllowSelfSignedServerCert;
if (!string.IsNullOrWhiteSpace(kafkaOptions?.TslClientCertPfxPathOrCertStoreSubject))
{
_selfSignedTrainMode = kafkaOptions.TslSelfSignedTrainMode;
_clientCert = GetClientCert(kafkaOptions.TslClientCertPfxPathOrCertStoreSubject, kafkaOptions.TslClientCertStoreFriendlyName, kafkaOptions?.TslClientCertPassword);
_processNetworkstreamTasksAction = ProcessNetworkstreamTasksTsl;
}
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
private async Task Producer()
{
await _asyncCollection.AddAsync(7);
await _asyncCollection.AddAsync(13);
_asyncCollection.CompleteAdding();
}
public void CompletedCollectionShouldPreventMoreItemsAdded()
{
var collection = new AsyncCollection <int>();
collection.Add(1);
collection.CompleteAdding();
collection.Add(1);
}
public void CompletedCollectionShouldShowCompletedTrue()
{
var collection = new AsyncCollection <int>();
collection.Add(1);
Assert.That(collection.IsCompleted, Is.False);
collection.CompleteAdding();
Assert.That(collection.IsCompleted, Is.True);
}
public async Task ConsumeEmpty()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var MyTask = Consumer(MyCollection);
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public void TryAddCompleted()
{ //****************************************
var MyCollection = new AsyncCollection <int>(1);
//****************************************
_ = MyCollection.CompleteAdding();
//****************************************
Assert.IsFalse(MyCollection.TryAdd(42), "Add unexpectedly succeeded");
}
/// <summary>
/// Stops the producer from accepting new messages, and optionally waits for in-flight messages to be sent before returning.
/// </summary>
/// <param name="waitForRequestsToComplete">True to wait for in-flight requests to complete, false otherwise.</param>
/// <param name="maxWait">Maximum time to wait for in-flight requests to complete. Has no effect if <c>waitForRequestsToComplete</c> is false</param>
public void Stop(bool waitForRequestsToComplete = true, TimeSpan?maxWait = null)
{
//block incoming data
_asyncCollection.CompleteAdding();
if (waitForRequestsToComplete)
{
//wait for the collection to drain
_postTask.Wait(maxWait ?? TimeSpan.FromSeconds(MaxDisposeWaitSeconds));
}
_stopToken.Cancel();
}
public void TakeAsyncFromMultipleThreads()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
int t1Counter = 0;
int t2Counter = 0;
var t1 = Task.Run(async() =>
{
while (!collection.IsCompleted)
{
try
{
await collection.TakeAsync();
t1Counter++;
}
catch (InvalidOperationException)
{
}
}
});
var t2 = Task.Run(async() =>
{
while (!collection.IsCompleted)
{
try
{
await collection.TakeAsync();
t2Counter++;
}
catch (InvalidOperationException)
{
}
}
});
for (int i = 0; i < 1000; i++)
{
collection.Add(i);
}
collection.CompleteAdding();
t1.Wait();
t2.Wait();
Assert.IsTrue(t1Counter > 400);
Assert.IsTrue(t2Counter > 400);
}
public async Task ConsumePreFilled()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var ProducerTask = Producer(MyCollection, 20, (index) => index);
var MyTask = Consumer(MyCollection);
await ProducerTask;
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public async Task ConsumeLimitPreFilled()
{ //****************************************
var MyCollection = new AsyncCollection <int>(10);
//****************************************
var ProducerTask = Producer(MyCollection, 100, (index) => index);
var MyTask = ConsumerWithWait(MyCollection, TimeSpan.Zero);
await ProducerTask;
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public async Task Consume()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var MyTask = Consumer(MyCollection);
foreach (var Item in Enumerable.Range(1, 10))
{
await MyCollection.Add(Item);
}
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public async Task ConsumeLimit()
{ //****************************************
var MyCollection = new AsyncCollection <int>(10);
//****************************************
var MyTask = ConsumerWithWait(MyCollection, TimeSpan.Zero);
foreach (var Item in Enumerable.Range(1, 100))
{
await MyCollection.Add(Item);
}
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
private static void StartProducer()
{
Task.Factory.StartNew(async() =>
{
var counter = 0;
while (++counter <= 50)
{
await asyncCollection.AddAsync(counter);
await asyncCollection.AddAsync(++counter);
Thread.Sleep(TimeSpan.FromSeconds(0.1));
}
asyncCollection.CompleteAdding();
});
}
async void Example14()
{
//限流
AsyncCollection <int> _asyncStack = new AsyncCollection <int>(new ConcurrentStack <int>(), maxCount: 1);
await _asyncStack.AddAsync(7); //这个添加过程会立即完成。
await _asyncStack.AddAsync(13); //这个添加（异步地）等待，直到7被移除，然后才会加入13。
_asyncStack.CompleteAdding();
//多个消费者
while (true)
{
//var takeResult = await _asyncStack.TryTaskAsync();
//if (!takeResult.Success) break;
//Trace.WriteLine(takeResult.Item);
}
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Run(async () => { await DedicatedSocketTask(); });
_disposeTask = _disposeToken.Token.CreateTask();
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan?maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, int maxRetry, TimeSpan?maximumReconnectionTimeout = null, StatisticsTrackerOptions statisticsTrackerOptions = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_maxRetry = maxRetry;
_statisticsTrackerOptions = statisticsTrackerOptions;
_sendTaskQueue = new AsyncCollection <SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection <SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Run(async() => { await DedicatedSocketTask(); });
_disposeTask = _disposeToken.Token.CreateTask();
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
public async Task AddMaximumThenComplete()
{ //****************************************
var MyCollection = new AsyncCollection <int>(1);
//****************************************
await MyCollection.Add(42);
var MyTask = MyCollection.Add(84);
_ = MyCollection.CompleteAdding();
//****************************************
Assert.IsFalse(await MyTask);
Assert.AreEqual(1, MyCollection.Count, "Count not as expected");
Assert.IsTrue(MyCollection.IsAddingCompleted, "Adding not completed");
Assert.IsFalse(MyCollection.IsCompleted, "Collection completed");
}
public async Task TakeAddThenComplete()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var MyTask = MyCollection.Take();
await MyCollection.Add(42);
_ = MyCollection.CompleteAdding();
var MyResult = await MyTask;
//****************************************
Assert.AreEqual(0, MyCollection.Count, "Count not as expected");
Assert.AreEqual(42, MyResult, "Result not as expected");
Assert.IsTrue(MyCollection.IsAddingCompleted, "Adding not completed");
Assert.IsTrue(MyCollection.IsCompleted, "Collection not completed");
}
async void Example13()
{
AsyncCollection <int> _asyncStack = new AsyncCollection <int>(new ConcurrentStack <int>()); //后进先出（栈）
AsyncCollection <int> _asyncBag = new AsyncCollection <int>(new ConcurrentBag <int>()); //无序（包）
//在栈的项目次序上有竞态条件。单线程中如果先运行生存者代码，后运行消费者代码，那项目的次序就像一个普通的栈
//生存者代码
await _asyncStack.AddAsync(7);
await _asyncStack.AddAsync(13);
_asyncStack.CompleteAdding();
//消费者代码
//先显示“13”，后显示“7”
while (await _asyncStack.OutputAvailableAsync())
{
Trace.WriteLine(_asyncStack.TakeAsync());
}
//当生产者和消费者都并发运行时（这是常见情况），消费者总是会得到最近加入的项目。这导致这个集合从整体上看不像是一个栈。当然了，包是根本没有次序的。
}
/// <summary>
/// Called by the <see cref="Code"/> implementation to check if the client wants to intercept a G/M/T-code
/// </summary>
/// <param name="code">Code to intercept</param>
/// <returns>True if the code has been resolved</returns>
/// <exception cref="OperationCanceledException">Code has been cancelled</exception>
private async Task <bool> Intercept(Code code)
{
// Avoid race conditions. A client can deal with only one code at once!
using (await _lock.LockAsync(Program.CancelSource.Token))
{
// Send it to the IPC client
await _codeQueue.AddAsync(code);
// Keep on processing commands from the interceptor until a handling result is returned.
// This must be either a Cancel, Ignore, or Resolve instruction!
try
{
if (await _commandQueue.OutputAvailableAsync(Program.CancelSource.Token))
{
BaseCommand command = await _commandQueue.TakeAsync(Program.CancelSource.Token);
// Code is cancelled. This invokes an OperationCanceledException on the code's task.
if (command is Cancel)
{
throw new OperationCanceledException();
}
// Code is resolved with a given result and the request is acknowledged
if (command is Resolve resolveCommand)
{
code.Result = (resolveCommand.Content == null) ? new CodeResult() : new CodeResult(resolveCommand.Type, resolveCommand.Content);
return(true);
}
// Code is ignored. Don't do anything
}
}
catch (Exception e) when(!(e is OperationCanceledException))
{
_codeQueue.CompleteAdding();
Console.WriteLine($"[err] Interception handler enocuntered an exception: {e}");
}
}
return(false);
}
async Task Test()
{
var _asyncStack = new AsyncCollection <int>(
new ConcurrentStack <int>());
var _asyncBag = new AsyncCollection <int>(
new ConcurrentBag <int>());
// Producer code
await _asyncStack.AddAsync(7);
await _asyncStack.AddAsync(13);
_asyncStack.CompleteAdding();
// Consumer code
// Displays "13" followed by "7".
while (await _asyncStack.OutputAvailableAsync())
{
Trace.WriteLine(await _asyncStack.TakeAsync());
}
}
/// <summary>
/// Called by the <see cref="Code"/> implementation to check if the client wants to intercept a G/M/T-code
/// </summary>
/// <param name="code">Code to intercept</param>
/// <returns>True if the code has been resolved</returns>
/// <exception cref="OperationCanceledException">Code has been cancelled</exception>
private async Task <bool> Intercept(Code code)
{
// Send it to the IPC client
await _codeQueue.AddAsync(code);
// Keep on processing commands from the interceptor until a handling result is returned.
// This must be either a Cancel, Ignore, or Resolve instruction!
try
{
if (await _commandQueue.OutputAvailableAsync(Program.CancellationToken))
{
BaseCommand command = await _commandQueue.TakeAsync(Program.CancellationToken);
// Code is cancelled. This invokes an OperationCanceledException on the code's task.
if (command is Cancel)
{
throw new OperationCanceledException();
}
// Code is resolved with a given result and the request is acknowledged
if (command is Resolve resolveCommand)
{
code.Result = (resolveCommand.Content == null) ? new CodeResult() : new CodeResult(resolveCommand.Type, resolveCommand.Content);
return(true);
}
// Code is ignored. Don't do anything
}
}
catch (Exception e) when(!(e is OperationCanceledException))
{
_codeQueue.CompleteAdding();
Connection.Logger.Error(e, "Interception processor caught an exception");
}
return(false);
}
public void Stop()
{
collection.CompleteAdding();
collectionForReentrantItems.CompleteAdding();
}
public void Stop()
{
_monitor.Stop();
processingQueue.CompleteAdding();
}
async Task Main(string[] args)
{
//不可变集合是永远不会改变的集合,写入操作会返回新实例，不可变集合之间通常共享了大部分存储空间，浪费不大。多个线程访问安全
///不可变栈
var list = ImmutableStack <int> .Empty;
list = list.Push(13);
list = list.Push(15);
foreach (var item in list)
{
Console.WriteLine(item);
}
///不可变队列 ImmutableQueue
//不可变列表
//支持索引、不经常修改、可以被多个线程安全访问
var immutlist = ImmutableList <int> .Empty;
immutlist = immutlist.Insert(0, 13);
immutlist = immutlist.Insert(0, 7);
//不可变set集合
//不需要存放重复内容，不经常修改，可以被多个线程安全访问
//ImmutableHashSet 不含重复元素的集合
//ImmutableSortedSet 已排序不含重复元素的集合
//不可变字典
//ImmutableSortedDictionary
//ImmutableDictionary
//线程安全集合是可同时被多个线程修改的可变集合，线程安全集合混合使用了细粒度锁定和无锁技术,优点是多个线程可安全地对其进行访问
//线程安全字典
//需要有一个键/值集合，多个线程同时读写时仍能保持同步
//ConcurrentDictionary
var dictionary = new ConcurrentDictionary <int, string>();
//第一个委托把本来的键0转换成值zero,第二个委托把键0和原来的值转换成字典中修改后的值 zero，只有字典中已民存在这个键时，最后一个委托才会运行
dictionary[0] = "zero";
var newValue = dictionary.AddOrUpdate(0, key => "Zero", (key, oldValue) => "Zero1");
dictionary.TryGetValue(0, out string currentalue);
Console.WriteLine(currentalue);
dictionary.TryRemove(0, out string removeValue);
//生产费消费者模型
//阻塞队列
//需要有一个管道,在进行之间传递消息或数据，例如一个线程下大装载数据，装载的同时把数据压进管道，与此同时，另一个线程在管道的接收端接收处理数据
//BlockingCollection 类可当做这种管道，阻塞队列，先进先出 限流 bounedCapacity属性
//不过如果用到这个的话，更推荐数据流
var blockqueue = new BlockingCollection <int>();
var blockqueueTask = Task.Factory.StartNew(() =>
{
blockqueue.Add(7);
blockqueue.Add(8);
blockqueue.Add(9);
blockqueue.CompleteAdding();
});
foreach (var item in blockqueue.GetConsumingEnumerable())
{
Console.WriteLine(item);
}
await blockqueueTask;
//阻塞栈和包
//首先有一个管道，在线程之间传递消息或数据，但不想（不需要）这个管道使用先进先出的语义
//blockingCollection 可以在创建时选择规则
var _blockingStack = new BlockingCollection <int>(new ConcurrentBag <int>());
//异步队列
//在代码的各个部分之间以选进先出的方式传递消息或数据 多个消费者时需要注意捕获InvalidOperationException异常
var _syncQueue = new BufferBlock <int>();
await _syncQueue.SendAsync(7);
await _syncQueue.SendAsync(13);
_syncQueue.Complete();
while (await _syncQueue.OutputAvailableAsync())
{
Console.WriteLine(await _syncQueue.ReceiveAsync());
}
//异步栈和包
//需要有一个管道，在程序的各个部分传递数据，但不希望先进先出
var _asyncStack = new AsyncCollection <int>(new ConcurrentBag <int>(), maxCount: 1);
//这个添加操作会立即完成，下一个添加会等待7被移除后
await _asyncStack.AddAsync(7);
_asyncStack.CompleteAdding();
while (await _asyncStack.OutputAvailableAsync())
{
var taskReuslt = await _asyncStack.TryTakeAsync();
if (taskReuslt.Success)
{
Console.WriteLine(taskReuslt.Item);
}
}
//阻塞/异步队列
//先进先出 足够灵活 同步或异步方式处理
var queue = new BufferBlock <int>();
await queue.SendAsync(1);
queue.Complete();
while (await queue.OutputAvailableAsync())
{
Console.WriteLine(await queue.ReceiveAsync());
}
Console.WriteLine("Hello World!");
}
private void stopProcessing()
{
taskQueue.CompleteAdding();
}
public void CompletedCollectionShouldShowCompletedTrue()
{
var collection = new AsyncCollection<int>();
collection.Add(1);
Assert.That(collection.IsCompleted, Is.False);
collection.CompleteAdding();
Assert.That(collection.IsCompleted, Is.True);
}
public void CompletedCollectionShouldPreventMoreItemsAdded()
{
var collection = new AsyncCollection<int>();
collection.Add(1);
collection.CompleteAdding();
collection.Add(1);
}