public static Task MergeAsync <T>(
IEnumerable <ChannelReader <T> > inputs,
ChannelWriter <T> output,
CancellationToken stoppingToken)
{
if (inputs is null)
{
throw new ArgumentNullException(nameof(inputs));
}
if (output is null)
{
throw new ArgumentNullException(nameof(output));
}
return(Task.Run(async() =>
{
try
{
await Task.WhenAll(inputs.Select(input => RedirectAsync(input, output, stoppingToken))
.ToArray())
.ConfigureAwait(false);
output.Complete();
}
catch (Exception ex)
{
output.Complete(ex);
throw;
}
},
stoppingToken));
}
/// <summary>
/// Asynchronously writes all entries from the source to the channel.
/// </summary>
/// <typeparam name="T">The input type of the channel.</typeparam>
/// <param name="target">The channel to write to.</param>
/// <param name="source">The asynchronous source data to use.</param>
/// <param name="complete">If true, will call .Complete() if all the results have successfully been written (or the source is emtpy).</param>
/// <param name="deferredExecution">If true, calls await Task.Yield() before writing.</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
/// <returns>A task containing the count of items written that completes when all the data has been written to the channel writer.
/// The count should be ignored if the number of iterations could exceed the max value of long.</returns>
public static async ValueTask <long> WriteAllAsync <T>(this ChannelWriter <T> target,
IAsyncEnumerable <T> source, bool complete = false, bool deferredExecution = false, CancellationToken cancellationToken = default)
{
if (target is null)
{
throw new ArgumentNullException(nameof(target));
}
if (source is null)
{
throw new ArgumentNullException(nameof(source));
}
Contract.EndContractBlock();
await target
.WaitToWriteAndThrowIfClosedAsync(ChannelClosedMessage, deferredExecution, cancellationToken)
.ConfigureAwait(false);
try
{
long count = 0;
var next = new ValueTask();
await foreach (var value in source)
{
await next.ConfigureAwait(false);
count++;
next = target.WriteAsync(value, cancellationToken);
}
await next.ConfigureAwait(false);
return(count);
}
catch (Exception ex)
{
if (complete)
{
target.Complete(ex);
complete = false;
}
throw;
}
finally
{
if (complete)
{
target.Complete();
}
}
}
public void Close()
{
lock (_semaphore)
{
if (_closed || _socket == null)
{
return;
}
else
{
try
{
_channelWriter.Complete();
_writerTask?.GetAwaiter().GetResult();
}
catch
{
// ignore, we are closing anyway
}
try
{
_socket.Close();
}
catch
{
// ignore, we are closing anyway
}
finally
{
_closed = true;
}
}
}
}
private async Task SortChunks(ChannelReader <List <T> > channelReader, ChannelWriter <List <T> > channelWriter)
{
try
{
int index = Interlocked.Increment(ref _concurrencyCounter);
_logger.Info($"{index}. Started sorting phase...");
var stopwatch = Stopwatch.StartNew();
while (await channelReader.WaitToReadAsync().ConfigureAwait(false))
{
if (channelReader.TryRead(out var chunk))
{
_logger.Debug($"{index}. Started sorting chunk ...");
chunk.Sort();
_logger.Debug($"{index}. Chunk was sorted in {stopwatch.Elapsed}");
stopwatch.Restart();
await channelWriter.WriteAsync(chunk).ConfigureAwait(false);
}
}
_logger.Info($"{index}. Sorting phase is completed.");
}
catch (Exception ex)
{
_logger.ErrorException("Error occured while sorting: ", ex);
channelWriter.Complete(ex);
throw;
}
}
private void ReadLines(string logPath, ChannelWriter <LogDatum> writer)
{
WriteMessage("Reading log file: {0}", logPath);
var sw = Stopwatch.StartNew();
int count = 0;
using (var fs = File.Open(logPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
using (var sr = new StreamReader(fs))
{
string line;
while ((line = sr.ReadLine()) != null)
{
count++;
if (line == String.Empty)
{
continue;
}
var logLine = new LogDatum(line, count);
writer.TryWrite(logLine);
}
}
writer.Complete(); // We won't do this if this is a live file that's still being written to (how do we tell? Do we need to know?)
sw.Stop();
WriteMessage($"Done reading log file. {count,10:N0} lines {sw.Elapsed} elapsed");
}
public void Stop()
{
_queueConsumer.Stop();
_inboundChannelWritter.Complete();
_inboundQueueProcessor.Wait();
_inboundChannelProcessor.Wait();
}
public async Task RunAsync(CancellationToken ct)
{
try
{
var rnd = new Random();
while (!ct.IsCancellationRequested)
{
++_count;
string message = $"Message{_count}";
Console.WriteLine($"Producing {message}");
await _writer.WriteAsync(message, ct);
Console.WriteLine($"Produced {message}");
await Task.Delay(TimeSpan.FromMilliseconds(rnd.Next(500, 1000)), ct);
}
}
catch (OperationCanceledException)
{
// Consider cancellation as normal exit
}
finally
{
Console.WriteLine("Producer marking writer complete");
_writer.Complete();
}
}
private static async Task RunAsync()
{
var manejadorZip = new ManejadorZip();
var manejadorImagenes = new ManejadorImagenes();
IEnumerable <string> listaDeFotografias = manejadorImagenes.GetListadoFotografias();
Channel <byte[]> m_channel = Channel.CreateUnbounded <byte[]>();
ChannelReader <byte[]> reader = m_channel.Reader;
await Task.Factory.StartNew(async() => {
while (await reader.WaitToReadAsync())
{
while (reader.TryRead(out byte[] miniatura))
{
await manejadorZip.AgregarAZipAsync(miniatura);
}
}
});
ChannelWriter <byte[]> writer = m_channel.Writer;
foreach (var foto in listaDeFotografias)
{
// Generar miniatura
byte[] miniatura = await manejadorImagenes.GenerarMiniaturaAsync(foto);
writer.TryWrite(miniatura);
}
writer.Complete();
}
public void Close()
{
lock (_semaphore)
{
if (!_closed)
{
try
{
_channelWriter.Complete();
_writerTask.GetAwaiter().GetResult();
}
catch (Exception)
{
}
try
{
_socket.Close();
}
catch (Exception)
{
// ignore, we are closing anyway
}
finally
{
_closed = true;
}
}
}
}
private static async Task StartMessagePumpAsync(Client client, ChannelWriter <bool> startSyncChannel, CancellationToken cancellationToken)
{
using var heartbeatTickerTicker = new TimerTicker(HeartbeatInterval);
var heartbeatChannel = heartbeatTickerTicker.GetChannel(() => new Tick(), cancellationToken);
startSyncChannel.Complete();
var channels = new[] { heartbeatChannel, };
while (!cancellationToken.IsCancellationRequested)
{
await foreach (var msg in ChannelExtension.Merge(channels, cancellationToken).ReadAllAsync(cancellationToken))
{
switch (msg)
{
case Tick:
client.WriteMessage("heartbeat", HeartbeatInterval);
continue;
default:
throw new NotSupportedException();
}
}
}
}
void Test()
{
Channel <int> queue = Channel.CreateBounded <int>(10);
// Producer code
ChannelWriter <int> writer = queue.Writer;
Task.Run(async() =>
{
await writer.WriteAsync(7);
await writer.WriteAsync(13);
writer.Complete();
}).GetAwaiter().GetResult();
// Consumer code
ChannelReader <int> reader = queue.Reader;
Task.Run(async() =>
{
while (await reader.WaitToReadAsync())
{
while (reader.TryRead(out int value))
{
Trace.WriteLine(value);
}
}
}).GetAwaiter().GetResult();
}
private async Task MapLeavesToIdsAsync(
ChannelReader <CatalogLeafItem> leafChannel,
ChannelWriter <string> packageIdChannel,
CancellationToken cancellationToken)
{
var seen = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
while (await leafChannel.WaitToReadAsync(cancellationToken))
{
while (leafChannel.TryRead(out var leaf))
{
// Skip package IDs that have already been seen.
if (!seen.Add(leaf.PackageId))
{
continue;
}
if (!packageIdChannel.TryWrite(leaf.PackageId))
{
await packageIdChannel.WriteAsync(leaf.PackageId);
}
}
}
packageIdChannel.Complete();
}
private async Task WriteItemsAsync(
ChannelWriter <int> writer,
int count,
int delay)
{
Exception localException = null;
try
{
for (var i = 0; i < count; i++)
{
await writer.WriteAsync(i);
await Task.Delay(delay);
}
}
catch (Exception ex)
{
localException = ex;
}
finally
{
writer.Complete(localException);
}
}
private async Task ReceiveMessagesAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
var message = await ReceiveMessageAsync(cancellationToken);
_log.LogTrace($"Enqueue {message}");
await _writer.WriteAsync(message, cancellationToken);
}
catch (TaskCanceledException ex)
{
_log.LogDebug(ex.Message);
}
catch (Exception ex)
{
_log.LogError(ex, ex.Message);
}
}
_writer.Complete();
_log.LogInformation($"Stopped receiving messages.");
}
/// <summary>
/// Asynchronously writes all entries from the source to the channel.
/// </summary>
/// <typeparam name="T">The input type of the channel.</typeparam>
/// <param name="target">The channel to write to.</param>
/// <param name="source">The asynchronous source data to use.</param>
/// <param name="complete">If true, will call .Complete() if all the results have successfully been written (or the source is emtpy).</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
/// <returns>A task containing the count of items written that completes when all the data has been written to the channel writer.
/// The count should be ignored if the number of iterations could exceed the max value of long.</returns>
public static async ValueTask <long> WriteAllAsync <T>(this ChannelWriter <T> target,
IEnumerable <ValueTask <T> > source, bool complete = false, CancellationToken cancellationToken = default)
{
await target.WaitToWriteAndThrowIfClosedAsync(
"The target channel was closed before writing could begin.",
cancellationToken);
long count = 0;
var next = new ValueTask();
foreach (var e in source)
{
var value = await e.ConfigureAwait(false);
await next.ConfigureAwait(false);
count++;
next = target.WriteAsync(value, cancellationToken);
}
await next.ConfigureAwait(false);
if (complete)
{
target.Complete();
}
return(count);
}
private void FetchPagesInParallel(int pageCount, ChannelWriter <IEnumerable <Person> > channelWriter, CancellationToken ct)
{
Task[] tasks = new Task[pageCount];
for (int page = 0; page < pageCount; ++page)
{
int localPage = page;
tasks[page] = _repo.GetPersonPage(page, _pageSize, ct)
.ContinueWith(task =>
{
if (task.IsCompletedSuccessfully)
{
Console.WriteLine("Page {0} fetched successfully.", localPage);
channelWriter.TryWrite(task.Result);
}
else
{
Console.WriteLine("Fetching page {0} failed: {1}.", localPage, task.Exception);
}
}, ct);
}
Task.WaitAll(tasks, ct);
// complete the channel so that the reader stops after reading all data
channelWriter.Complete();
}
private async ValueTask StartReadPipe(PipeReader pipeReader, CancellationToken cancellationToken)
{
var processor = new PacketsProcessor(_channelWriter);
try
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var result = await pipeReader.ReadAsync(cancellationToken);
var length = await processor.ProcessBytesAsync(result.Buffer, cancellationToken);
pipeReader.AdvanceTo(result.Buffer.GetPosition(length));
if (result.IsCompleted)
{
break;
}
}
}
catch (SocketException e)
{
await pipeReader.CompleteAsync(e);
_channelWriter.Complete(e);
}
}
private async Task ReadLines(string logPath, ChannelWriter <LogDatum> writer)
{
WriteMessage($"Starting channel reader - ReadLines");
_sw.Start();
int totalCount = 0;
using (var fs = LogFile.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
using (var sr = new StreamReader(fs))
{
// This sets up our infinite loop of reading the file for new changes (until cancelled)
while (!CancelSource.IsCancellationRequested)
{
var count = ReadCurrentSetOfFileLines(sr, writer);
// var count = await ReadCurrentSetOfFileLinesAsync(sr, writer);
totalCount += count;
try
{
// Console.Write(".");
await Task.Delay(500, CancelSource.Token); // Ensures early exit if cancelled
}
catch (Exception ex) when(ex is ChannelClosedException || ex is OperationCanceledException || ex is TaskCanceledException)
{
WriteMessage($"ERROR: ReadLines - {ex.Message}, {ex.GetType()}");
}
}
}
WriteMessage($"Total Lines read: {totalCount:N0}");
// We've been cancelled at this point, close out the channel
writer.Complete();
WriteMessage($"Finished channel reader - ReadLines");
}
public async Task SplitAsync(long maxSize, CancellationToken token)
{
var partSize = 0L;
var lines = new List <string>();
string line;
using var reader = new StreamReader(_fileName);
while ((line = await reader.ReadLineAsync()) != null)
{
partSize += line.Length * sizeof(char);
if (partSize >= maxSize)
{
await _fileWriter.WriteAsync(new ListPart(lines.ToArray()), token);
partSize = 0;
lines.Clear();
}
lines.Add(line);
}
if (partSize < maxSize)
{
await _fileWriter.WriteAsync(new ListPart(lines.ToArray()), token);
}
_fileWriter.Complete();
}
public void Close()
{
if (IsAlive())
{
writer.Complete();
}
}
private async Task WriteItemsAsync(
ChannelWriter <int> writer,
int count,
int delay,
CancellationToken cancellationToken)
{
Exception localException = null;
try
{
for (var i = 0; i < count; i++)
{
await writer.WriteAsync(i, cancellationToken);
// Use the cancellationToken in other APIs that accept cancellation
// tokens so the cancellation can flow down to them.
await Task.Delay(delay, cancellationToken);
}
}
catch (Exception ex)
{
localException = ex;
}
writer.Complete(localException);
}
async Task StreamingWriter(ChannelWriter <IDictionary <string, object> > writer, IDictionary <string, object> sentData, int count, int streamItemWaiting)
{
Exception localException = null;
try
{
for (var i = 0; i < count; i++)
{
sentData[SignalRConstants.Timestamp] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString();
await writer.WriteAsync(sentData);
if (streamItemWaiting > 0)
{
await Task.Delay(streamItemWaiting);
}
// Update statistics
SignalRUtils.RecordSend(sentData, StatisticsCollector);
}
}
catch (Exception ex)
{
localException = ex;
}
writer.Complete(localException);
}
private async Task OutboundMessageProcessor()
{
try
{
while (await _outboundChannelReader.WaitToReadAsync())
{
while (_outboundChannelReader.TryRead(out byte[] data))
{
try
{
_outboundQueueChannelWritter.TryWrite(data);
_outboundFileChannelWritter.TryWrite(data);
}
catch (Exception ex)
{
Console.WriteLine(ex);
//TODO
throw;
}
}
}
_outboundQueueChannelWritter.Complete();
_outboundFileChannelWritter.Complete();
}
catch (Exception ex)
{
Console.WriteLine(ex);
//TODO
throw;
}
}
private async Task WriteStreamItemsAsync(
string id,
ChannelWriter <string> writer,
int delay,
CancellationToken cancellationToken)
{
var input = _hubBuffer.BufferChannel[id];
Console.WriteLine($"Process the incoming stream from {id}");
Exception localException = null;
try
{
while (await input.WaitToReadAsync())
{
while (input.TryRead(out var item))
{
Console.WriteLine(item);
await writer.WriteAsync(item);
await Task.Delay(delay);
}
}
}
catch (Exception ex)
{
localException = ex;
}
writer.Complete(localException);
}
private async Task WriteItemsAsync(
ChannelWriter <int> writer,
int count,
int delay,
CancellationToken cancellationToken)
{
Exception localException = null;
try
{
for (var i = 0; i < count; i++)
{
// Check the cancellation token regularly so that the server will stop
// producing items if the client disconnects.
cancellationToken.ThrowIfCancellationRequested();
await writer.WriteAsync(i);
// Use the cancellationToken in other APIs that accept cancellation
// tokens so the cancellation can flow down to them.
await Task.Delay(delay, cancellationToken);
}
}
catch (Exception ex)
{
localException = ex;
}
finally
{
writer.Complete(localException);
}
}
private async void SortLines(ChannelReader <ILine> reader, ChannelWriter <ILine> writer)
{
var lines = new Dictionary <int, ILine>();
while (await reader.WaitToReadAsync())
{
while (reader.TryRead(out var line))
{
if (_parserCount == 1)
{
writer.TryWrite(line);
}
else
{
lines.Add(line.LogLine.LineNumber, line);
if (lines.Count >= SortBatchSize * 2)
{
SortBatch(lines, writer);
}
}
}
}
// Do the remainder of the lines
if (_parserCount != 1)
{
SortBatch(lines, writer, true);
}
writer.Complete();
}
static async ChannelReader <string> Stage1(ChannelWriter <string> ch)
{
await ch.WriteAsync("Stage 1 - message 1");
await ch.WriteAsync("Stage 1 - message 2");
ch.Complete();
}
private async Task ReadChunks(IEnumerable <T> source, ChannelWriter <List <T> > channelWriter)
{
try
{
_logger.Info("Starting reading phase...");
var stopwatch = Stopwatch.StartNew();
var chunkStopwatch = Stopwatch.StartNew();
var currentChunk = new List <T>();
long currentChunkSize = 0;
long currentChunkNumber = 0;
foreach (var value in source)
{
currentChunk.Add(value);
if (currentChunkSize < _chunkSize)
{
currentChunkSize++;
}
else
{
_logger.Debug($"Chunk {currentChunkNumber} with {currentChunk.Count} lines was readen in {chunkStopwatch.Elapsed}.");
chunkStopwatch.Restart();
await channelWriter.WriteAsync(currentChunk).ConfigureAwait(false);
currentChunk = new List <T>();
currentChunkSize = 0;
currentChunkNumber++;
}
}
_logger.Debug($"Chunk {currentChunkNumber} with {currentChunk.Count} lines was readen in {chunkStopwatch.Elapsed}.");
await channelWriter.WriteAsync(currentChunk).ConfigureAwait(false);
channelWriter.Complete();
stopwatch.Stop();
_logger.Info($"Reading phase is completed in {stopwatch.Elapsed}.");
}
catch (Exception ex)
{
_logger.ErrorException("Error occured while reading: ", ex);
channelWriter.Complete(ex);
throw;
}
}
public void SetCommand(string command)
{
_inputWriter.WriteAsync(command);
if (command == "exit")
{
_inputWriter.Complete();
}
}
private static async Task ReadFromClient(Socket socket, ChannelWriter <double> channel, int bufferSize)
{
var endPoint = socket.RemoteEndPoint; // save remote end point for logging
Logger.Trace("read from client {} started", endPoint);
using (var stream = new NetworkStream(socket))
{
var buffer = new byte[bufferSize];
var current = new List <byte>();
while (true)
{
var readCount = await stream.ReadAsync(buffer);
if (readCount == 0)
{
break;
}
var newLineIndex = Array.IndexOf(buffer, (byte)'\n', 0, readCount);
if (newLineIndex == -1)
{
// not find new line
current.AddRange(buffer.Take(readCount));
Logger.Trace("get {} bytes unfinished data from client {}", readCount, socket.RemoteEndPoint);
}
else
{
// find new line
current.AddRange(buffer.Take(newLineIndex));
var stringData = Encoding.UTF8.GetString(current.ToArray());
Logger.Trace("get string with length {} from client {}: {}",
stringData.Length,
socket.RemoteEndPoint,
stringData);
current.Clear();
if (readCount > newLineIndex + 1)
{
// more data in this read
current.AddRange(buffer.Take(readCount).Skip(newLineIndex + 1)); // skip '\n'
Logger.Trace("get {} bytes unfinished data from client {}",
readCount - newLineIndex - 1,
socket.RemoteEndPoint);
}
if (!double.TryParse(stringData, out var result))
{
result = double.NaN;
}
await channel.WriteAsync(result);
}
}
channel.Complete();
}
Logger.Trace("end read from client {}", endPoint);
}