/// <summary>
/// Runs a repeated parallel operation
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="source">The channel where requests are read from</param>
/// <param name="handler">The method to invoke for each item</param>
/// <param name="token">Token.</param>
/// <param name="maxparallel">The maximum parallelism to use.</param>
/// <param name="errorHandler">The error handler</param>
/// <typeparam name="T">The type of data elements to handle</typeparam>
public static Task RunParallelAsync <T>(IReadChannel <T> source, Func <T, Task> handler, System.Threading.CancellationToken token, int maxparallel = 10, Func <T, Exception, Task> errorHandler = null)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (maxparallel <= 0)
{
throw new ArgumentOutOfRangeException(nameof(maxparallel), maxparallel, "The size of the queue must be greater than zero");
}
return(AutomationExtensions.RunTask(
new
{
Requests = source
},
async self =>
{
using (var tp = new TaskPool <T>(maxparallel, errorHandler))
while (true)
{
if (token.IsCancellationRequested)
{
throw new TaskCanceledException();
}
await tp.Run(await source.ReadAsync(), handler).ConfigureAwait(false);
}
}));
}
/// <summary>
/// Initializes a new instance of the <see cref="T:CoCoL.Blocks.Prefix`1"/> class.
/// </summary>
/// <param name="input">The input channel.</param>
/// <param name="output">The output channel.</param>
/// <param name="value">The initial value to emit.</param>
/// <param name="repeat">The number of copies to initially emit.</param>
public Prefix(IReadChannel <T> input, IWriteChannel <T> output, T value, long repeat = 1)
{
m_input = input ?? throw new ArgumentNullException(nameof(input));
m_output = output ?? throw new ArgumentNullException(nameof(output));
m_value = value;
m_repeat = repeat;
}
/// <summary>
/// Sets up a simple process that distributes the input to the output channels
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="input">The data source.</param>
/// <param name="output">The channels to distribute the data to.</param>
/// <typeparam name="T">The 1st type parameter.</typeparam>
public static Task ScatterAsync <T>(IReadChannel <T> input, IWriteChannel <T>[] output, ScatterGatherPolicy policy = ScatterGatherPolicy.Any)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
if (output == null || output.Any(x => x == null))
{
throw new ArgumentNullException(nameof(output));
}
return(AutomationExtensions.RunTask(
new { Input = input, Output = output },
async self =>
{
if (policy == ScatterGatherPolicy.Any)
{
while (true)
{
await MultiChannelAccess.WriteToAnyAsync(await self.Input.ReadAsync(), self.Output);
}
}
else
{
var ix = 0;
while (true)
{
await self.Output[ix].WriteAsync(await self.Input.ReadAsync());
ix = (ix + 1) % output.Length;
}
}
}
));
}
/// <summary>
/// Wraps the channel with a latency hidign instance, if required by config
/// </summary>
/// <returns>The buffered read channel.</returns>
/// <param name="input">The input channel.</param>
private static IReadChannel <T> AsBufferedRead(IReadChannel <T> input)
{
if (Config.NetworkChannelLatencyBufferSize != 0 && input is NetworkChannel <T> && (Config.AllChannelsNetworked || ((INamedItem)input).Name == "tick"))
{
return(new LatencyHidingReader <T>(input, Config.NetworkChannelLatencyBufferSize));
}
return(input);
}
/// <summary>
/// Wraps the channel with a latency hidign instance, if required by config
/// </summary>
/// <returns>The buffered read channel.</returns>
/// <param name="input">The input channel.</param>
private static IReadChannel <T> AsBufferedRead <T>(IReadChannel <T> input)
{
if (Config.NetworkChannelLatencyBufferSize != 0 && input is NetworkChannel <T> && Config.NetworkedChannels)
{
return(new LatencyHidingReader <T>(input, Config.NetworkChannelLatencyBufferSize));
}
return(input);
}
/// <summary>
/// Reads the channel in a probing and asynchronous manner
/// </summary>
/// <param name="self">The channel to read from</param>
/// <typeparam name="T">The channel data type parameter.</typeparam>
/// <returns>True if the read succeeded, false otherwise</returns>
public static Task <KeyValuePair <bool, T> > TryReadAsync <T>(this IReadChannel <T> self)
{
return(self.ReadAsync(Timeout.Immediate, null).ContinueWith(x => {
if (x.IsFaulted || x.IsCanceled)
{
return new KeyValuePair <bool, T>(false, default(T));
}
return new KeyValuePair <bool, T>(true, x.Result);
}));
}
/// <summary>
/// Initializes a new instance of the <see cref="CoCoL.ChannelMarkerWrapper<T>"/> class.
/// </summary>
/// <param name="attribute">The attribute describing the channel.</param>
public ChannelMarkerWrapper(ChannelNameAttribute attribute)
{
if (attribute == null)
{
throw new ArgumentNullException("attribute");
}
Attribute = attribute;
ForWrite = ChannelMarker.ForWrite <T>(attribute);
ForRead = ChannelMarker.ForRead <T>(attribute);
}
/// <summary>
/// Reads the channel in a probing and asynchronous manner
/// </summary>
/// <param name="self">The channel to read from</param>
/// <param name="waittime">The amount of time to wait before cancelling</param>
/// <typeparam name="T">The channel data type parameter.</typeparam>
/// <returns>True if the read succeeded, false otherwise</returns>
public static Task <KeyValuePair <bool, T> > TryReadAsync <T>(this IReadChannel <T> self, TimeSpan waittime)
{
return(self.ReadAsync(new TimeoutOffer(waittime)).ContinueWith(x => {
if (x.IsFaulted || x.IsCanceled)
{
return new KeyValuePair <bool, T>(false, default(T));
}
return new KeyValuePair <bool, T>(true, x.Result);
}));
}
public Prefix(IReadChannel<T> input, IWriteChannel<T> output, T value, long repeat = 1)
{
if (input == null)
throw new ArgumentNullException("input");
if (output == null)
throw new ArgumentNullException("output");
m_input = input;
m_output = output;
m_value = value;
m_repeat = repeat;
}
/// <summary>
/// Sets up a simple channel transformation process
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="input">The input channel.</param>
/// <param name="output">The output channel.</param>
/// <param name="transform">The method transforming the input</param>
/// <param name="maxparallel">The maximum number of parallel handlers</param>
/// <typeparam name="TIn">The input type parameter.</typeparam>
/// <typeparam name="TOut">The output type parameter.</typeparam>
public static Task TransformAsync <TIn, TOut>(IReadChannel <TIn> input, IWriteChannel <TOut> output, Func <TIn, Task <TOut> > transform, int maxparallel = 1)
{
if (output == null)
{
throw new ArgumentNullException(nameof(output));
}
if (transform == null)
{
throw new ArgumentNullException(nameof(transform));
}
return(CollectAsync(input, async data => await output.WriteAsync(await transform(data)), maxparallel));
}
public Successor(IReadChannel <long> input, IWriteChannel <long> output)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
m_input = input;
m_output = output;
}
public Identity(IReadChannel <T> input, IWriteChannel <T> output)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
m_input = input;
m_output = output;
}
/// <summary>
/// Read from the channel in a probing manner
/// </summary>
/// <param name="self">The channel to read from</param>
/// <param name="result">The read result</param>
/// <typeparam name="T">The channel data type parameter.</typeparam>
/// <returns>True if the read succeeded, false otherwise</returns>
public static bool TryRead <T>(this IReadChannel <T> self, out T result)
{
var res = self.ReadAsync(new TimeoutOffer(Timeout.Immediate)).WaitForTask();
if (res.IsFaulted || res.IsCanceled)
{
result = default(T);
return(false);
}
else
{
result = res.Result;
return(true);
}
}
/// <summary>
/// Runs the identity process, which simply forwards a value.
/// </summary>
/// <param name="chan_read">The channel to read from</param>
/// <param name="chan_write">The channel to write to</param>
private static async void RunIdentity(IReadChannel <T> chan_read, IWriteChannel <T> chan_write)
{
try
{
while (true)
{
await chan_write.WriteAsync(await chan_read.ReadAsync());
}
}
catch (RetiredException)
{
chan_read.Retire();
chan_write.Retire();
}
}
/// <summary>
/// Read from the channel in a blocking manner
/// </summary>
/// <param name="self">The channel to read from</param>
/// <param name="timeout">The maximum time to wait for a value</param>
/// <returns>>The value read from the channel</returns>
/// <typeparam name="T">The channel data type parameter.</typeparam>
public static T Read <T>(this IReadChannel <T> self, TimeSpan timeout)
{
try
{
return(self.ReadAsync(new TimeoutOffer(timeout)).WaitForTask().Result);
}
catch (AggregateException aex)
{
if (aex.Flatten().InnerExceptions.Count == 1)
{
throw aex.InnerException;
}
throw;
}
}
/// <summary>
/// Constructs a new runner
/// </summary>
/// <param name="options">The options to use</param>
/// <param name="reqchan">The request channel</param>
/// <param name="respchan">The response channel</param>
/// <param name="response">The expected response string</param>
public WebRequestWorker(IWorkerOptions options, IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan, string response)
//: base(options, maximumrequests, response)
: base(options, reqchan, respchan, response)
{
foreach (var h in options.Headers)
{
var parts = h?.Split(':', 2);
if (parts == null || parts.Length < 2)
{
throw new ArgumentException($"Bad header format: {h}");
}
m_headers.Add(parts[0], parts[1]);
}
m_data = string.IsNullOrEmpty(options.Body) ? null : System.Text.Encoding.UTF8.GetBytes(options.Body);
m_initialized.TrySetResult(true);
}
/// <summary>
/// Read from the channel set in a blocking manner
/// </summary>
/// <param name="self">The channels to read from</param>
/// <param name="channel">The channel written to</param>
/// <param name="timeout">The maximum time to wait for a value</param>
/// <typeparam name="T">The channel data type parameter.</typeparam>
/// <returns>The value read from a channel</returns>
public static T ReadFromAny <T>(this MultiChannelSetRead <T> self, out IReadChannel <T> channel, TimeSpan timeout)
{
try
{
var res = self.ReadFromAnyAsync(timeout).WaitForTask().Result;
channel = res.Channel;
return(res.Value);
}
catch (AggregateException aex)
{
if (aex.Flatten().InnerExceptions.Count == 1)
{
throw aex.InnerException;
}
throw;
}
}
public Delta(IReadChannel <T> input, IWriteChannel <T> outputA, IWriteChannel <T> outputB)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (outputA == null)
{
throw new ArgumentNullException("outputA");
}
if (outputB == null)
{
throw new ArgumentNullException("outputB");
}
m_input = input;
m_outputA = outputA;
m_outputB = outputB;
}
/// <summary>
/// Constructs a new runner
/// </summary>
/// <param name="options">The options to use</param>
/// <param name="reqchan">The request channel</param>
/// <param name="respchan">The response channel</param>
/// <param name="response">The expected response string</param>
public CurlWorker(IWorkerOptions options, IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan, string response)
//: base(options, maximumrequests, response)
: base(options, reqchan, respchan, response)
{
var sb = new StringBuilder();
sb.Append($"--silent --show-error --request {EscapeAndQuoteArgument(options.Verb)} {EscapeAndQuoteArgument(options.RequestUrl)}");
foreach (var h in options.Headers)
{
sb.Append($" --header {EscapeAndQuoteArgument(h)}");
}
if (!string.IsNullOrEmpty(options.Body))
{
sb.Append($" --data {EscapeAndQuoteArgument(options.Body)}");
}
m_commandline = sb.ToString();
m_initialized.TrySetResult(true);
}
/// <summary>
/// A process that reads numbers and discards those that are
/// divisible by a certain number and forwards the rest
/// </summary>
/// <returns>The awaitable task that represents the process</returns>
/// <param name="number">The number used to test and filter divisible numbers with.</param>
/// <param name="input">The channel where data is read from.</param>
/// <param name="output">The channel where non-multiple values are written to.</param>
private static async Task RunNoMultiplesAsync(long number, IReadChannel <long> input, IWriteChannel <long> output)
{
try
{
while (true)
{
var v = await input.ReadAsync();
if (v % number != 0)
{
await output.WriteAsync(v);
}
}
}
catch (RetiredException)
{
input.Retire();
output.Retire();
}
}
/// <summary>
/// Constructs a new runner
/// </summary>
/// <param name="options">The options to use</param>
/// <param name="reqchan">The request channel</param>
/// <param name="respchan">The response channel</param>
/// <param name="response">The expected response string</param>
public SocketWorker(IWorkerOptions options, IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan, string response)
: base(options, reqchan, respchan, response)
{
var sb = new StringBuilder();
var uri = new Uri(options.RequestUrl);
sb.Append($"{options.Verb} {uri.PathAndQuery} HTTP/1.1{CRLF}");
sb.Append($"Host: {uri.Host}{CRLF}");
foreach (var h in options.Headers)
{
sb.Append($"{h}{CRLF}");
}
sb.Append(CRLF);
if (!string.IsNullOrEmpty(options.Body))
{
sb.Append(options.Body);
}
m_data = System.Text.Encoding.UTF8.GetBytes(sb.ToString());
m_initialized.TrySetResult(true);
}
/// <summary>
/// Runs the delta process, which copies the value it reads onto two different channels
/// </summary>
/// <param name="chan_read">The channel to read from</param>
/// <param name="chan_a">The channel to write to</param>
/// <param name="chan_b">The channel to write to</param>
private static async void RunDelta(IReadChannel <T> chan_read, IWriteChannel <T> chan_a, IWriteChannel <T> chan_b)
{
try
{
while (true)
{
var value = await chan_read.ReadAsync();
await Task.WhenAll(
chan_a.WriteAsync(value),
chan_b.WriteAsync(value)
);
}
}
catch (RetiredException)
{
chan_read.Retire();
chan_a.Retire();
chan_b.Retire();
}
}
/// <summary>
/// Runs the delta process, which copies the value it reads onto two different channels
/// </summary>
/// <param name="chan_read">The channel to read from</param>
/// <param name="chan_a">The channel to write to</param>
/// <param name="chan_b">The channel to write to</param>
private static async void RunDeltaAlt(IReadChannel <T> chan_read, IWriteChannel <T> chan_a, IWriteChannel <T> chan_b)
{
try
{
while (true)
{
var value = await chan_read.ReadAsync();
var offer = new SingleOffer <T>();
await Task.WhenAll(
chan_a.WriteAsync(value),
chan_b.WriteAsync(value, CoCoL.Timeout.Infinite, offer)
);
}
}
catch (RetiredException)
{
chan_read.Retire();
chan_a.Retire();
chan_b.Retire();
}
}
/// <summary>
/// A process that spawns a new set of processes
/// when a number is received.
/// By inserting a NoMultiples into the chain,
/// it is guaranteed that no numbers that are divisible
/// with any number in the chain can be retrieved by the
/// Sieve.
/// </summary>
/// <returns>The awaitable task that represents the process</returns>
/// <param name="input">The channel to read numbers from</param>
/// <param name="output">The channel to write numbers to</param>
private static async Task RunSieveAsync(IReadChannel <long> input, IWriteChannel <long> output)
{
var chan = ChannelManager.CreateChannel <long>();
try
{
var n = await input.ReadAsync();
await output.WriteAsync(n);
await Task.WhenAll(
RunNoMultiplesAsync(n, input, chan),
RunSieveAsync(chan, output)
);
}
catch (RetiredException)
{
chan.Retire();
input.Retire();
output.Retire();
}
}
/// <summary>
/// Combines inputs from <paramref name="inputA"/> and <paramref name="inputB"/> and writes it to <paramref name="output"/>.
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="inputA">One input channel.</param>
/// <param name="inputB">Another input channel.</param>
/// <param name="output">The output result.</param>
/// <param name="method">The zip method</param>
/// <typeparam name="Ta">One data type parameter.</typeparam>
/// <typeparam name="Tb">Another data type parameter.</typeparam>
/// <typeparam name="TOut">The result type parameter.</typeparam>
public static Task ZipAsync <Ta, Tb, TOut>(IReadChannel <Ta> inputA, IReadChannel <Tb> inputB, IWriteChannel <TOut> output, Func <Ta, Tb, Task <TOut> > method)
{
if (inputA == null)
{
throw new ArgumentNullException(nameof(inputA));
}
if (inputB == null)
{
throw new ArgumentNullException(nameof(inputB));
}
if (output == null)
{
throw new ArgumentNullException(nameof(output));
}
if (method == null)
{
throw new ArgumentNullException(nameof(method));
}
return(AutomationExtensions.RunTask(
new
{
InputA = inputA,
InputB = inputB,
Output = output
},
async self =>
{
while (true)
{
var readA = self.InputA.ReadAsync();
var readB = self.InputB.ReadAsync();
var vA = await readA;
var vB = await readB;
await output.WriteAsync(await method(vA, vB));
}
}
));
}
/// <summary>
/// Gets a read channel from a marker interface.
/// </summary>
/// <returns>The requested channel.</returns>
/// <param name="channel">The marker interface, or a real channel instance.</param>
/// <typeparam name="T">The channel type.</typeparam>
public static IReadChannelEnd <T> GetChannel <T>(IReadChannel <T> channel)
{
var rt = channel as ReadMarker <T>;
if (rt == null)
{
return(channel.AsReadOnly());
}
var scope = ChannelScope.Current;
if (rt.Attribute.TargetScope == ChannelNameScope.Parent)
{
scope = scope.ParentScope;
}
else if (rt.Attribute.TargetScope == ChannelNameScope.Global)
{
scope = ChannelScope.Root;
}
return(GetChannel <T>(rt.Attribute, scope).AsReadOnly());
}
/// <summary>
/// Splits all values from a channel into two channels
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="input">The channel to read from.</param>
/// <param name="outputA">One output channel.</param>
/// <param name="outputB">Another output channel.</param>
/// <param name="method">The method that performs the split.</param>
/// <typeparam name="TIn">The input data type parameter.</typeparam>
/// <typeparam name="Ta">One output data type parameter.</typeparam>
/// <typeparam name="Tb">Another output data type parameter.</typeparam>
public static Task SplitAsync <TIn, Ta, Tb>(IReadChannel <TIn> input, IWriteChannel <Ta> outputA, IWriteChannel <Tb> outputB, Func <TIn, Task <Tuple <Ta, Tb> > > method)
{
if (input == null)
{
throw new ArgumentNullException(nameof(input));
}
if (outputA == null)
{
throw new ArgumentNullException(nameof(outputA));
}
if (outputB == null)
{
throw new ArgumentNullException(nameof(outputB));
}
if (method == null)
{
throw new ArgumentNullException(nameof(method));
}
return(AutomationExtensions.RunTask(
new
{
Input = input,
OutputA = outputA,
OutputB = outputB
},
async self =>
{
while (true)
{
var n = await method(await self.Input.ReadAsync());
var writeA = self.OutputA.WriteAsync(n.Item1);
await self.OutputB.WriteAsync(n.Item2);
await writeA;
}
}
));
}
/// <summary>
/// The runner helper method that calls the abstract request method
/// </summary>
/// <returns>An awaitable task</returns>
protected Task RunAsync(IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan)
{
return(AutomationExtensions.RunTask(
new { reqchan, respchan },
async _ =>
{
while (await reqchan.ReadAsync())
{
var start = DateTime.Now;
try
{
var resp = await PeformRequestAsync();
await respchan.WriteAsync(new RequestResult()
{
Started = start,
Finished = DateTime.Now,
Failed = m_expectedresponse != null && m_expectedresponse != resp
});
}
catch (System.Exception ex)
{
await respchan.WriteAsync(new RequestResult()
{
Started = start,
Finished = DateTime.Now,
Failed = true,
Exception = ex
});
if (m_options.Verbose)
{
Console.WriteLine(ex.Message);
}
}
}
}));
}
/// <summary>
/// Constructs a new runner
/// </summary>
/// <param name="options">The options to use</param>
/// <param name="reqchan">The request channel</param>
/// <param name="respchan">The response channel</param>
/// <param name="response">The expected response string</param>
public RunnerBase(IWorkerOptions options, IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan, string response)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (string.IsNullOrWhiteSpace(options.RequestUrl) || !System.Uri.TryCreate(options.RequestUrl, UriKind.Absolute, out var _))
{
throw new ArgumentException($"Invalid request url: {options.RequestUrl}", nameof(options.RequestUrl));
}
if (string.IsNullOrWhiteSpace(options.Verb))
{
throw new ArgumentException(nameof(options.Verb), "Missing HTTP verb");
}
m_id = System.Threading.Interlocked.Increment(ref _id);
m_options = options;
m_expectedresponse = response;
if (m_expectedresponse != null)
{
Result = RunAsync(reqchan, respchan);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="T:CoCoL.Blocks.Delta`1"/> process.
/// </summary>
/// <param name="input">The input channel.</param>
/// <param name="outputA">Output channel A.</param>
/// <param name="outputB">Output channel B.</param>
public Delta(IReadChannel <T> input, IWriteChannel <T> outputA, IWriteChannel <T> outputB)
{
m_input = input ?? throw new ArgumentNullException(nameof(input));
m_outputA = outputA ?? throw new ArgumentNullException(nameof(outputA));
m_outputB = outputB ?? throw new ArgumentNullException(nameof(outputB));
}
